Mercurial > hg > nginx
annotate src/core/ngx_slab.c @ 8045:aa28c802409f
Resolver: make TCP write timer event cancelable.
Similar to 70e65bf8dfd7, the change is made to ensure that the ability to
cancel resolver tasks is fully controlled by the caller. As mentioned in the
referenced commit, it is safe to make this timer cancelable because resolve
tasks can have their own timeouts that are not cancelable.
The scenario where this may become a problem is a periodic background resolve
task (not tied to a specific request or a client connection), which receives a
response with short TTL, large enough to warrant fallback to a TCP query.
With each event loop wakeup, we either have a previously set write timer
instance or schedule a new one. The non-cancelable write timer can delay or
block graceful shutdown of a worker even if the ngx_resolver_ctx_t->cancelable
flag is set by the API user, and there are no other tasks or connections.
We use the resolver API in this way to maintain the list of upstream server
addresses specified with the 'resolve' parameter, and there could be third-party
modules implementing similar logic.
author | Aleksei Bavshin <a.bavshin@f5.com> |
---|---|
date | Wed, 01 Jun 2022 20:17:23 -0700 |
parents | d97d09ef3afe |
children |
rev | line source |
---|---|
441
da8c5707af39
nginx-0.1.0-2004-09-28-12:34:51 import; set copyright and remove unused files
Igor Sysoev <igor@sysoev.ru>
parents:
357
diff
changeset
|
1 |
da8c5707af39
nginx-0.1.0-2004-09-28-12:34:51 import; set copyright and remove unused files
Igor Sysoev <igor@sysoev.ru>
parents:
357
diff
changeset
|
2 /* |
444
42d11f017717
nginx-0.1.0-2004-09-29-20:00:49 import; remove years from copyright
Igor Sysoev <igor@sysoev.ru>
parents:
441
diff
changeset
|
3 * Copyright (C) Igor Sysoev |
4412 | 4 * Copyright (C) Nginx, Inc. |
441
da8c5707af39
nginx-0.1.0-2004-09-28-12:34:51 import; set copyright and remove unused files
Igor Sysoev <igor@sysoev.ru>
parents:
357
diff
changeset
|
5 */ |
da8c5707af39
nginx-0.1.0-2004-09-28-12:34:51 import; set copyright and remove unused files
Igor Sysoev <igor@sysoev.ru>
parents:
357
diff
changeset
|
6 |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
7 #include <ngx_config.h> |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
8 #include <ngx_core.h> |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
9 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
10 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
11 #define NGX_SLAB_PAGE_MASK 3 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
12 #define NGX_SLAB_PAGE 0 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
13 #define NGX_SLAB_BIG 1 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
14 #define NGX_SLAB_EXACT 2 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
15 #define NGX_SLAB_SMALL 3 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
16 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
17 #if (NGX_PTR_SIZE == 4) |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
18 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
19 #define NGX_SLAB_PAGE_FREE 0 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
20 #define NGX_SLAB_PAGE_BUSY 0xffffffff |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
21 #define NGX_SLAB_PAGE_START 0x80000000 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
22 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
23 #define NGX_SLAB_SHIFT_MASK 0x0000000f |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
24 #define NGX_SLAB_MAP_MASK 0xffff0000 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
25 #define NGX_SLAB_MAP_SHIFT 16 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
26 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
27 #define NGX_SLAB_BUSY 0xffffffff |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
28 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
29 #else /* (NGX_PTR_SIZE == 8) */ |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
30 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
31 #define NGX_SLAB_PAGE_FREE 0 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
32 #define NGX_SLAB_PAGE_BUSY 0xffffffffffffffff |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
33 #define NGX_SLAB_PAGE_START 0x8000000000000000 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
34 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
35 #define NGX_SLAB_SHIFT_MASK 0x000000000000000f |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
36 #define NGX_SLAB_MAP_MASK 0xffffffff00000000 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
37 #define NGX_SLAB_MAP_SHIFT 32 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
38 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
39 #define NGX_SLAB_BUSY 0xffffffffffffffff |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
40 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
41 #endif |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
42 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
43 |
6823
88c8c3d65184
Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents:
6809
diff
changeset
|
44 #define ngx_slab_slots(pool) \ |
88c8c3d65184
Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents:
6809
diff
changeset
|
45 (ngx_slab_page_t *) ((u_char *) (pool) + sizeof(ngx_slab_pool_t)) |
88c8c3d65184
Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents:
6809
diff
changeset
|
46 |
88c8c3d65184
Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents:
6809
diff
changeset
|
47 #define ngx_slab_page_type(page) ((page)->prev & NGX_SLAB_PAGE_MASK) |
88c8c3d65184
Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents:
6809
diff
changeset
|
48 |
88c8c3d65184
Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents:
6809
diff
changeset
|
49 #define ngx_slab_page_prev(page) \ |
88c8c3d65184
Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents:
6809
diff
changeset
|
50 (ngx_slab_page_t *) ((page)->prev & ~NGX_SLAB_PAGE_MASK) |
88c8c3d65184
Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents:
6809
diff
changeset
|
51 |
88c8c3d65184
Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents:
6809
diff
changeset
|
52 #define ngx_slab_page_addr(pool, page) \ |
88c8c3d65184
Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents:
6809
diff
changeset
|
53 ((((page) - (pool)->pages) << ngx_pagesize_shift) \ |
88c8c3d65184
Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents:
6809
diff
changeset
|
54 + (uintptr_t) (pool)->start) |
88c8c3d65184
Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents:
6809
diff
changeset
|
55 |
88c8c3d65184
Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents:
6809
diff
changeset
|
56 |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
57 #if (NGX_DEBUG_MALLOC) |
1771
b794d78e52c5
take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents:
1645
diff
changeset
|
58 |
4224
4c4997de65b8
Using of junk value in slab allocator similar to modern FreeBSD values.
Igor Sysoev <igor@sysoev.ru>
parents:
4223
diff
changeset
|
59 #define ngx_slab_junk(p, size) ngx_memset(p, 0xA5, size) |
1771
b794d78e52c5
take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents:
1645
diff
changeset
|
60 |
4764
0ed8088f43b4
Reduced the number of preprocessor directives.
Ruslan Ermilov <ru@nginx.com>
parents:
4412
diff
changeset
|
61 #elif (NGX_HAVE_DEBUG_MALLOC) |
1771
b794d78e52c5
take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents:
1645
diff
changeset
|
62 |
b794d78e52c5
take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents:
1645
diff
changeset
|
63 #define ngx_slab_junk(p, size) \ |
4224
4c4997de65b8
Using of junk value in slab allocator similar to modern FreeBSD values.
Igor Sysoev <igor@sysoev.ru>
parents:
4223
diff
changeset
|
64 if (ngx_debug_malloc) ngx_memset(p, 0xA5, size) |
1771
b794d78e52c5
take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents:
1645
diff
changeset
|
65 |
b794d78e52c5
take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents:
1645
diff
changeset
|
66 #else |
b794d78e52c5
take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents:
1645
diff
changeset
|
67 |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
68 #define ngx_slab_junk(p, size) |
1771
b794d78e52c5
take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents:
1645
diff
changeset
|
69 |
b794d78e52c5
take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents:
1645
diff
changeset
|
70 #endif |
b794d78e52c5
take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents:
1645
diff
changeset
|
71 |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
72 static ngx_slab_page_t *ngx_slab_alloc_pages(ngx_slab_pool_t *pool, |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
73 ngx_uint_t pages); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
74 static void ngx_slab_free_pages(ngx_slab_pool_t *pool, ngx_slab_page_t *page, |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
75 ngx_uint_t pages); |
2611
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
76 static void ngx_slab_error(ngx_slab_pool_t *pool, ngx_uint_t level, |
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
77 char *text); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
78 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
79 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
80 static ngx_uint_t ngx_slab_max_size; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
81 static ngx_uint_t ngx_slab_exact_size; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
82 static ngx_uint_t ngx_slab_exact_shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
83 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
84 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
85 void |
7081
69f9ee0342db
Slab: fixed initialization on win32.
Ruslan Ermilov <ru@nginx.com>
parents:
7050
diff
changeset
|
86 ngx_slab_sizes_init(void) |
69f9ee0342db
Slab: fixed initialization on win32.
Ruslan Ermilov <ru@nginx.com>
parents:
7050
diff
changeset
|
87 { |
69f9ee0342db
Slab: fixed initialization on win32.
Ruslan Ermilov <ru@nginx.com>
parents:
7050
diff
changeset
|
88 ngx_uint_t n; |
69f9ee0342db
Slab: fixed initialization on win32.
Ruslan Ermilov <ru@nginx.com>
parents:
7050
diff
changeset
|
89 |
69f9ee0342db
Slab: fixed initialization on win32.
Ruslan Ermilov <ru@nginx.com>
parents:
7050
diff
changeset
|
90 ngx_slab_max_size = ngx_pagesize / 2; |
69f9ee0342db
Slab: fixed initialization on win32.
Ruslan Ermilov <ru@nginx.com>
parents:
7050
diff
changeset
|
91 ngx_slab_exact_size = ngx_pagesize / (8 * sizeof(uintptr_t)); |
69f9ee0342db
Slab: fixed initialization on win32.
Ruslan Ermilov <ru@nginx.com>
parents:
7050
diff
changeset
|
92 for (n = ngx_slab_exact_size; n >>= 1; ngx_slab_exact_shift++) { |
69f9ee0342db
Slab: fixed initialization on win32.
Ruslan Ermilov <ru@nginx.com>
parents:
7050
diff
changeset
|
93 /* void */ |
69f9ee0342db
Slab: fixed initialization on win32.
Ruslan Ermilov <ru@nginx.com>
parents:
7050
diff
changeset
|
94 } |
69f9ee0342db
Slab: fixed initialization on win32.
Ruslan Ermilov <ru@nginx.com>
parents:
7050
diff
changeset
|
95 } |
69f9ee0342db
Slab: fixed initialization on win32.
Ruslan Ermilov <ru@nginx.com>
parents:
7050
diff
changeset
|
96 |
69f9ee0342db
Slab: fixed initialization on win32.
Ruslan Ermilov <ru@nginx.com>
parents:
7050
diff
changeset
|
97 |
69f9ee0342db
Slab: fixed initialization on win32.
Ruslan Ermilov <ru@nginx.com>
parents:
7050
diff
changeset
|
98 void |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
99 ngx_slab_init(ngx_slab_pool_t *pool) |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
100 { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
101 u_char *p; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
102 size_t size; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
103 ngx_int_t m; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
104 ngx_uint_t i, n, pages; |
6823
88c8c3d65184
Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents:
6809
diff
changeset
|
105 ngx_slab_page_t *slots, *page; |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
106 |
6861
e4590dfd97ff
Win32: support 64-bit compilation with MSVC.
Maxim Dounin <mdounin@mdounin.ru>
parents:
6830
diff
changeset
|
107 pool->min_size = (size_t) 1 << pool->min_shift; |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
108 |
6823
88c8c3d65184
Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents:
6809
diff
changeset
|
109 slots = ngx_slab_slots(pool); |
88c8c3d65184
Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents:
6809
diff
changeset
|
110 |
88c8c3d65184
Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents:
6809
diff
changeset
|
111 p = (u_char *) slots; |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
112 size = pool->end - p; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
113 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
114 ngx_slab_junk(p, size); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
115 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
116 n = ngx_pagesize_shift - pool->min_shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
117 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
118 for (i = 0; i < n; i++) { |
6824
ea12328518dc
Slab: added comment about list heads.
Ruslan Ermilov <ru@nginx.com>
parents:
6823
diff
changeset
|
119 /* only "next" is used in list head */ |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
120 slots[i].slab = 0; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
121 slots[i].next = &slots[i]; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
122 slots[i].prev = 0; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
123 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
124 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
125 p += n * sizeof(ngx_slab_page_t); |
6828 | 126 |
127 pool->stats = (ngx_slab_stat_t *) p; | |
128 ngx_memzero(pool->stats, n * sizeof(ngx_slab_stat_t)); | |
129 | |
130 p += n * sizeof(ngx_slab_stat_t); | |
131 | |
132 size -= n * (sizeof(ngx_slab_page_t) + sizeof(ngx_slab_stat_t)); | |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
133 |
1645
31622d9f2c0d
the simple expression has the same precision without overflow
Igor Sysoev <igor@sysoev.ru>
parents:
1018
diff
changeset
|
134 pages = (ngx_uint_t) (size / (ngx_pagesize + sizeof(ngx_slab_page_t))); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
135 |
6823
88c8c3d65184
Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents:
6809
diff
changeset
|
136 pool->pages = (ngx_slab_page_t *) p; |
88c8c3d65184
Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents:
6809
diff
changeset
|
137 ngx_memzero(pool->pages, pages * sizeof(ngx_slab_page_t)); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
138 |
6823
88c8c3d65184
Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents:
6809
diff
changeset
|
139 page = pool->pages; |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
140 |
6824
ea12328518dc
Slab: added comment about list heads.
Ruslan Ermilov <ru@nginx.com>
parents:
6823
diff
changeset
|
141 /* only "next" is used in list head */ |
ea12328518dc
Slab: added comment about list heads.
Ruslan Ermilov <ru@nginx.com>
parents:
6823
diff
changeset
|
142 pool->free.slab = 0; |
ea12328518dc
Slab: added comment about list heads.
Ruslan Ermilov <ru@nginx.com>
parents:
6823
diff
changeset
|
143 pool->free.next = page; |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
144 pool->free.prev = 0; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
145 |
6823
88c8c3d65184
Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents:
6809
diff
changeset
|
146 page->slab = pages; |
88c8c3d65184
Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents:
6809
diff
changeset
|
147 page->next = &pool->free; |
88c8c3d65184
Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents:
6809
diff
changeset
|
148 page->prev = (uintptr_t) &pool->free; |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
149 |
6823
88c8c3d65184
Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents:
6809
diff
changeset
|
150 pool->start = ngx_align_ptr(p + pages * sizeof(ngx_slab_page_t), |
88c8c3d65184
Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents:
6809
diff
changeset
|
151 ngx_pagesize); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
152 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
153 m = pages - (pool->end - pool->start) / ngx_pagesize; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
154 if (m > 0) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
155 pages -= m; |
6823
88c8c3d65184
Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents:
6809
diff
changeset
|
156 page->slab = pages; |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
157 } |
2611
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
158 |
5718
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
159 pool->last = pool->pages + pages; |
6829
6e757036e588
Slab: free pages statistics.
Ruslan Ermilov <ru@nginx.com>
parents:
6828
diff
changeset
|
160 pool->pfree = pages; |
5718
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
161 |
5634
5024d29354f1
Core: slab log_nomem flag.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5362
diff
changeset
|
162 pool->log_nomem = 1; |
2611
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
163 pool->log_ctx = &pool->zero; |
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
164 pool->zero = '\0'; |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
165 } |
357
e260514b9ad4
nginx-0.0.7-2004-06-16-23:36:07 import
Igor Sysoev <igor@sysoev.ru>
parents:
diff
changeset
|
166 |
e260514b9ad4
nginx-0.0.7-2004-06-16-23:36:07 import
Igor Sysoev <igor@sysoev.ru>
parents:
diff
changeset
|
167 |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
168 void * |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
169 ngx_slab_alloc(ngx_slab_pool_t *pool, size_t size) |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
170 { |
966
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
171 void *p; |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
172 |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
173 ngx_shmtx_lock(&pool->mutex); |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
174 |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
175 p = ngx_slab_alloc_locked(pool, size); |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
176 |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
177 ngx_shmtx_unlock(&pool->mutex); |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
178 |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
179 return p; |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
180 } |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
181 |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
182 |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
183 void * |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
184 ngx_slab_alloc_locked(ngx_slab_pool_t *pool, size_t size) |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
185 { |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
186 size_t s; |
7049 | 187 uintptr_t p, m, mask, *bitmap; |
188 ngx_uint_t i, n, slot, shift, map; | |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
189 ngx_slab_page_t *page, *prev, *slots; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
190 |
5766
abdb027be9d5
Core: bugfix for the ngx_slab_max_size case.
Jianjun Zheng <codeeply@gmail.com>
parents:
5726
diff
changeset
|
191 if (size > ngx_slab_max_size) { |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
192 |
966
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
193 ngx_log_debug1(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0, |
967 | 194 "slab alloc: %uz", size); |
966
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
195 |
4829
40de49cf6b37
Fixed overflow if ngx_slab_alloc() is called with very big "size" argument.
Ruslan Ermilov <ru@nginx.com>
parents:
4764
diff
changeset
|
196 page = ngx_slab_alloc_pages(pool, (size >> ngx_pagesize_shift) |
40de49cf6b37
Fixed overflow if ngx_slab_alloc() is called with very big "size" argument.
Ruslan Ermilov <ru@nginx.com>
parents:
4764
diff
changeset
|
197 + ((size % ngx_pagesize) ? 1 : 0)); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
198 if (page) { |
6823
88c8c3d65184
Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents:
6809
diff
changeset
|
199 p = ngx_slab_page_addr(pool, page); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
200 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
201 } else { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
202 p = 0; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
203 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
204 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
205 goto done; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
206 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
207 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
208 if (size > pool->min_size) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
209 shift = 1; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
210 for (s = size - 1; s >>= 1; shift++) { /* void */ } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
211 slot = shift - pool->min_shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
212 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
213 } else { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
214 shift = pool->min_shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
215 slot = 0; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
216 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
217 |
6828 | 218 pool->stats[slot].reqs++; |
219 | |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
220 ngx_log_debug2(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0, |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
221 "slab alloc: %uz slot: %ui", size, slot); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
222 |
6823
88c8c3d65184
Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents:
6809
diff
changeset
|
223 slots = ngx_slab_slots(pool); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
224 page = slots[slot].next; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
225 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
226 if (page->next != page) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
227 |
966
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
228 if (shift < ngx_slab_exact_shift) { |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
229 |
6826
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
230 bitmap = (uintptr_t *) ngx_slab_page_addr(pool, page); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
231 |
7049 | 232 map = (ngx_pagesize >> shift) / (8 * sizeof(uintptr_t)); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
233 |
6826
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
234 for (n = 0; n < map; n++) { |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
235 |
6826
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
236 if (bitmap[n] != NGX_SLAB_BUSY) { |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
237 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
238 for (m = 1, i = 0; m; m <<= 1, i++) { |
6826
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
239 if (bitmap[n] & m) { |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
240 continue; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
241 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
242 |
6826
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
243 bitmap[n] |= m; |
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
244 |
7049 | 245 i = (n * 8 * sizeof(uintptr_t) + i) << shift; |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
246 |
6828 | 247 p = (uintptr_t) bitmap + i; |
248 | |
249 pool->stats[slot].used++; | |
250 | |
6826
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
251 if (bitmap[n] == NGX_SLAB_BUSY) { |
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
252 for (n = n + 1; n < map; n++) { |
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
253 if (bitmap[n] != NGX_SLAB_BUSY) { |
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
254 goto done; |
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
255 } |
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
256 } |
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
257 |
6823
88c8c3d65184
Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents:
6809
diff
changeset
|
258 prev = ngx_slab_page_prev(page); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
259 prev->next = page->next; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
260 page->next->prev = page->prev; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
261 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
262 page->next = NULL; |
6826
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
263 page->prev = NGX_SLAB_SMALL; |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
264 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
265 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
266 goto done; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
267 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
268 } |
6826
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
269 } |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
270 |
6826
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
271 } else if (shift == ngx_slab_exact_shift) { |
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
272 |
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
273 for (m = 1, i = 0; m; m <<= 1, i++) { |
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
274 if (page->slab & m) { |
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
275 continue; |
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
276 } |
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
277 |
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
278 page->slab |= m; |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
279 |
6826
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
280 if (page->slab == NGX_SLAB_BUSY) { |
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
281 prev = ngx_slab_page_prev(page); |
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
282 prev->next = page->next; |
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
283 page->next->prev = page->prev; |
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
284 |
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
285 page->next = NULL; |
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
286 page->prev = NGX_SLAB_EXACT; |
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
287 } |
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
288 |
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
289 p = ngx_slab_page_addr(pool, page) + (i << shift); |
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
290 |
6828 | 291 pool->stats[slot].used++; |
292 | |
6826
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
293 goto done; |
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
294 } |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
295 |
966
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
296 } else { /* shift > ngx_slab_exact_shift */ |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
297 |
6827 | 298 mask = ((uintptr_t) 1 << (ngx_pagesize >> shift)) - 1; |
299 mask <<= NGX_SLAB_MAP_SHIFT; | |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
300 |
6826
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
301 for (m = (uintptr_t) 1 << NGX_SLAB_MAP_SHIFT, i = 0; |
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
302 m & mask; |
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
303 m <<= 1, i++) |
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
304 { |
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
305 if (page->slab & m) { |
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
306 continue; |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
307 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
308 |
6826
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
309 page->slab |= m; |
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
310 |
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
311 if ((page->slab & NGX_SLAB_MAP_MASK) == mask) { |
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
312 prev = ngx_slab_page_prev(page); |
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
313 prev->next = page->next; |
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
314 page->next->prev = page->prev; |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
315 |
6826
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
316 page->next = NULL; |
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
317 page->prev = NGX_SLAB_BIG; |
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
318 } |
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
319 |
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
320 p = ngx_slab_page_addr(pool, page) + (i << shift); |
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
321 |
6828 | 322 pool->stats[slot].used++; |
323 | |
6826
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
324 goto done; |
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
325 } |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
326 } |
6826
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
327 |
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
328 ngx_slab_error(pool, NGX_LOG_ALERT, "ngx_slab_alloc(): page is busy"); |
d0404c9a7675
Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents:
6825
diff
changeset
|
329 ngx_debug_point(); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
330 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
331 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
332 page = ngx_slab_alloc_pages(pool, 1); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
333 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
334 if (page) { |
966
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
335 if (shift < ngx_slab_exact_shift) { |
6823
88c8c3d65184
Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents:
6809
diff
changeset
|
336 bitmap = (uintptr_t *) ngx_slab_page_addr(pool, page); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
337 |
6827 | 338 n = (ngx_pagesize >> shift) / ((1 << shift) * 8); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
339 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
340 if (n == 0) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
341 n = 1; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
342 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
343 |
6830
6eed5ed31e22
Slab: commented bitmap initialization for small allocations.
Ruslan Ermilov <ru@nginx.com>
parents:
6829
diff
changeset
|
344 /* "n" elements for bitmap, plus one requested */ |
7050
8c5e3cc21332
Slab: fixed small allocations on systems with large pagesize.
Ruslan Ermilov <ru@nginx.com>
parents:
7049
diff
changeset
|
345 |
8c5e3cc21332
Slab: fixed small allocations on systems with large pagesize.
Ruslan Ermilov <ru@nginx.com>
parents:
7049
diff
changeset
|
346 for (i = 0; i < (n + 1) / (8 * sizeof(uintptr_t)); i++) { |
8c5e3cc21332
Slab: fixed small allocations on systems with large pagesize.
Ruslan Ermilov <ru@nginx.com>
parents:
7049
diff
changeset
|
347 bitmap[i] = NGX_SLAB_BUSY; |
8c5e3cc21332
Slab: fixed small allocations on systems with large pagesize.
Ruslan Ermilov <ru@nginx.com>
parents:
7049
diff
changeset
|
348 } |
8c5e3cc21332
Slab: fixed small allocations on systems with large pagesize.
Ruslan Ermilov <ru@nginx.com>
parents:
7049
diff
changeset
|
349 |
8c5e3cc21332
Slab: fixed small allocations on systems with large pagesize.
Ruslan Ermilov <ru@nginx.com>
parents:
7049
diff
changeset
|
350 m = ((uintptr_t) 1 << ((n + 1) % (8 * sizeof(uintptr_t)))) - 1; |
8c5e3cc21332
Slab: fixed small allocations on systems with large pagesize.
Ruslan Ermilov <ru@nginx.com>
parents:
7049
diff
changeset
|
351 bitmap[i] = m; |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
352 |
7049 | 353 map = (ngx_pagesize >> shift) / (8 * sizeof(uintptr_t)); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
354 |
7050
8c5e3cc21332
Slab: fixed small allocations on systems with large pagesize.
Ruslan Ermilov <ru@nginx.com>
parents:
7049
diff
changeset
|
355 for (i = i + 1; i < map; i++) { |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
356 bitmap[i] = 0; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
357 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
358 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
359 page->slab = shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
360 page->next = &slots[slot]; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
361 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_SMALL; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
362 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
363 slots[slot].next = page; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
364 |
6828 | 365 pool->stats[slot].total += (ngx_pagesize >> shift) - n; |
366 | |
6827 | 367 p = ngx_slab_page_addr(pool, page) + (n << shift); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
368 |
6828 | 369 pool->stats[slot].used++; |
370 | |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
371 goto done; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
372 |
966
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
373 } else if (shift == ngx_slab_exact_shift) { |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
374 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
375 page->slab = 1; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
376 page->next = &slots[slot]; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
377 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_EXACT; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
378 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
379 slots[slot].next = page; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
380 |
7049 | 381 pool->stats[slot].total += 8 * sizeof(uintptr_t); |
6828 | 382 |
6823
88c8c3d65184
Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents:
6809
diff
changeset
|
383 p = ngx_slab_page_addr(pool, page); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
384 |
6828 | 385 pool->stats[slot].used++; |
386 | |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
387 goto done; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
388 |
966
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
389 } else { /* shift > ngx_slab_exact_shift */ |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
390 |
881
5ed08a4c3bd3
fix building on 64-bit platforms
Igor Sysoev <igor@sysoev.ru>
parents:
860
diff
changeset
|
391 page->slab = ((uintptr_t) 1 << NGX_SLAB_MAP_SHIFT) | shift; |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
392 page->next = &slots[slot]; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
393 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_BIG; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
394 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
395 slots[slot].next = page; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
396 |
6828 | 397 pool->stats[slot].total += ngx_pagesize >> shift; |
398 | |
6823
88c8c3d65184
Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents:
6809
diff
changeset
|
399 p = ngx_slab_page_addr(pool, page); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
400 |
6828 | 401 pool->stats[slot].used++; |
402 | |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
403 goto done; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
404 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
405 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
406 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
407 p = 0; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
408 |
6828 | 409 pool->stats[slot].fails++; |
410 | |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
411 done: |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
412 |
6480 | 413 ngx_log_debug1(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0, |
414 "slab alloc: %p", (void *) p); | |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
415 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
416 return (void *) p; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
417 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
418 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
419 |
5726
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
420 void * |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
421 ngx_slab_calloc(ngx_slab_pool_t *pool, size_t size) |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
422 { |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
423 void *p; |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
424 |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
425 ngx_shmtx_lock(&pool->mutex); |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
426 |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
427 p = ngx_slab_calloc_locked(pool, size); |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
428 |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
429 ngx_shmtx_unlock(&pool->mutex); |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
430 |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
431 return p; |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
432 } |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
433 |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
434 |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
435 void * |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
436 ngx_slab_calloc_locked(ngx_slab_pool_t *pool, size_t size) |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
437 { |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
438 void *p; |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
439 |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
440 p = ngx_slab_alloc_locked(pool, size); |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
441 if (p) { |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
442 ngx_memzero(p, size); |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
443 } |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
444 |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
445 return p; |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
446 } |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
447 |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
448 |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
449 void |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
450 ngx_slab_free(ngx_slab_pool_t *pool, void *p) |
357
e260514b9ad4
nginx-0.0.7-2004-06-16-23:36:07 import
Igor Sysoev <igor@sysoev.ru>
parents:
diff
changeset
|
451 { |
966
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
452 ngx_shmtx_lock(&pool->mutex); |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
453 |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
454 ngx_slab_free_locked(pool, p); |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
455 |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
456 ngx_shmtx_unlock(&pool->mutex); |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
457 } |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
458 |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
459 |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
460 void |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
461 ngx_slab_free_locked(ngx_slab_pool_t *pool, void *p) |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
462 { |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
463 size_t size; |
1018
06306a20b8bb
fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents:
1016
diff
changeset
|
464 uintptr_t slab, m, *bitmap; |
6828 | 465 ngx_uint_t i, n, type, slot, shift, map; |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
466 ngx_slab_page_t *slots, *page; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
467 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
468 ngx_log_debug1(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0, "slab free: %p", p); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
469 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
470 if ((u_char *) p < pool->start || (u_char *) p > pool->end) { |
2611
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
471 ngx_slab_error(pool, NGX_LOG_ALERT, "ngx_slab_free(): outside of pool"); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
472 goto fail; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
473 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
474 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
475 n = ((u_char *) p - pool->start) >> ngx_pagesize_shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
476 page = &pool->pages[n]; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
477 slab = page->slab; |
6823
88c8c3d65184
Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents:
6809
diff
changeset
|
478 type = ngx_slab_page_type(page); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
479 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
480 switch (type) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
481 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
482 case NGX_SLAB_SMALL: |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
483 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
484 shift = slab & NGX_SLAB_SHIFT_MASK; |
6861
e4590dfd97ff
Win32: support 64-bit compilation with MSVC.
Maxim Dounin <mdounin@mdounin.ru>
parents:
6830
diff
changeset
|
485 size = (size_t) 1 << shift; |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
486 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
487 if ((uintptr_t) p & (size - 1)) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
488 goto wrong_chunk; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
489 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
490 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
491 n = ((uintptr_t) p & (ngx_pagesize - 1)) >> shift; |
7049 | 492 m = (uintptr_t) 1 << (n % (8 * sizeof(uintptr_t))); |
493 n /= 8 * sizeof(uintptr_t); | |
5362
79b9101cecf4
Handling of ngx_int_t != intptr_t case.
Maxim Dounin <mdounin@mdounin.ru>
parents:
4829
diff
changeset
|
494 bitmap = (uintptr_t *) |
79b9101cecf4
Handling of ngx_int_t != intptr_t case.
Maxim Dounin <mdounin@mdounin.ru>
parents:
4829
diff
changeset
|
495 ((uintptr_t) p & ~((uintptr_t) ngx_pagesize - 1)); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
496 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
497 if (bitmap[n] & m) { |
6828 | 498 slot = shift - pool->min_shift; |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
499 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
500 if (page->next == NULL) { |
6823
88c8c3d65184
Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents:
6809
diff
changeset
|
501 slots = ngx_slab_slots(pool); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
502 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
503 page->next = slots[slot].next; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
504 slots[slot].next = page; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
505 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
506 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_SMALL; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
507 page->next->prev = (uintptr_t) page | NGX_SLAB_SMALL; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
508 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
509 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
510 bitmap[n] &= ~m; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
511 |
6827 | 512 n = (ngx_pagesize >> shift) / ((1 << shift) * 8); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
513 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
514 if (n == 0) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
515 n = 1; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
516 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
517 |
7050
8c5e3cc21332
Slab: fixed small allocations on systems with large pagesize.
Ruslan Ermilov <ru@nginx.com>
parents:
7049
diff
changeset
|
518 i = n / (8 * sizeof(uintptr_t)); |
8c5e3cc21332
Slab: fixed small allocations on systems with large pagesize.
Ruslan Ermilov <ru@nginx.com>
parents:
7049
diff
changeset
|
519 m = ((uintptr_t) 1 << (n % (8 * sizeof(uintptr_t)))) - 1; |
8c5e3cc21332
Slab: fixed small allocations on systems with large pagesize.
Ruslan Ermilov <ru@nginx.com>
parents:
7049
diff
changeset
|
520 |
8c5e3cc21332
Slab: fixed small allocations on systems with large pagesize.
Ruslan Ermilov <ru@nginx.com>
parents:
7049
diff
changeset
|
521 if (bitmap[i] & ~m) { |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
522 goto done; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
523 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
524 |
7049 | 525 map = (ngx_pagesize >> shift) / (8 * sizeof(uintptr_t)); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
526 |
7050
8c5e3cc21332
Slab: fixed small allocations on systems with large pagesize.
Ruslan Ermilov <ru@nginx.com>
parents:
7049
diff
changeset
|
527 for (i = i + 1; i < map; i++) { |
6828 | 528 if (bitmap[i]) { |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
529 goto done; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
530 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
531 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
532 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
533 ngx_slab_free_pages(pool, page, 1); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
534 |
6828 | 535 pool->stats[slot].total -= (ngx_pagesize >> shift) - n; |
536 | |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
537 goto done; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
538 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
539 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
540 goto chunk_already_free; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
541 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
542 case NGX_SLAB_EXACT: |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
543 |
1016
d0e8c81d3bb7
fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents:
967
diff
changeset
|
544 m = (uintptr_t) 1 << |
d0e8c81d3bb7
fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents:
967
diff
changeset
|
545 (((uintptr_t) p & (ngx_pagesize - 1)) >> ngx_slab_exact_shift); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
546 size = ngx_slab_exact_size; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
547 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
548 if ((uintptr_t) p & (size - 1)) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
549 goto wrong_chunk; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
550 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
551 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
552 if (slab & m) { |
6828 | 553 slot = ngx_slab_exact_shift - pool->min_shift; |
554 | |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
555 if (slab == NGX_SLAB_BUSY) { |
6823
88c8c3d65184
Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents:
6809
diff
changeset
|
556 slots = ngx_slab_slots(pool); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
557 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
558 page->next = slots[slot].next; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
559 slots[slot].next = page; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
560 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
561 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_EXACT; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
562 page->next->prev = (uintptr_t) page | NGX_SLAB_EXACT; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
563 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
564 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
565 page->slab &= ~m; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
566 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
567 if (page->slab) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
568 goto done; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
569 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
570 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
571 ngx_slab_free_pages(pool, page, 1); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
572 |
7049 | 573 pool->stats[slot].total -= 8 * sizeof(uintptr_t); |
6828 | 574 |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
575 goto done; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
576 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
577 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
578 goto chunk_already_free; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
579 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
580 case NGX_SLAB_BIG: |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
581 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
582 shift = slab & NGX_SLAB_SHIFT_MASK; |
6861
e4590dfd97ff
Win32: support 64-bit compilation with MSVC.
Maxim Dounin <mdounin@mdounin.ru>
parents:
6830
diff
changeset
|
583 size = (size_t) 1 << shift; |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
584 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
585 if ((uintptr_t) p & (size - 1)) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
586 goto wrong_chunk; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
587 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
588 |
1016
d0e8c81d3bb7
fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents:
967
diff
changeset
|
589 m = (uintptr_t) 1 << ((((uintptr_t) p & (ngx_pagesize - 1)) >> shift) |
d0e8c81d3bb7
fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents:
967
diff
changeset
|
590 + NGX_SLAB_MAP_SHIFT); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
591 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
592 if (slab & m) { |
6828 | 593 slot = shift - pool->min_shift; |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
594 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
595 if (page->next == NULL) { |
6823
88c8c3d65184
Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents:
6809
diff
changeset
|
596 slots = ngx_slab_slots(pool); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
597 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
598 page->next = slots[slot].next; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
599 slots[slot].next = page; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
600 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
601 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_BIG; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
602 page->next->prev = (uintptr_t) page | NGX_SLAB_BIG; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
603 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
604 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
605 page->slab &= ~m; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
606 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
607 if (page->slab & NGX_SLAB_MAP_MASK) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
608 goto done; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
609 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
610 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
611 ngx_slab_free_pages(pool, page, 1); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
612 |
6828 | 613 pool->stats[slot].total -= ngx_pagesize >> shift; |
614 | |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
615 goto done; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
616 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
617 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
618 goto chunk_already_free; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
619 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
620 case NGX_SLAB_PAGE: |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
621 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
622 if ((uintptr_t) p & (ngx_pagesize - 1)) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
623 goto wrong_chunk; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
624 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
625 |
6809
87d7e640b45d
Slab: improved double free detection.
Ruslan Ermilov <ru@nginx.com>
parents:
6808
diff
changeset
|
626 if (!(slab & NGX_SLAB_PAGE_START)) { |
2611
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
627 ngx_slab_error(pool, NGX_LOG_ALERT, |
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
628 "ngx_slab_free(): page is already free"); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
629 goto fail; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
630 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
631 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
632 if (slab == NGX_SLAB_PAGE_BUSY) { |
2611
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
633 ngx_slab_error(pool, NGX_LOG_ALERT, |
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
634 "ngx_slab_free(): pointer to wrong page"); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
635 goto fail; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
636 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
637 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
638 size = slab & ~NGX_SLAB_PAGE_START; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
639 |
7457
d97d09ef3afe
Slab: removed redundant page calculation (ticket #1721).
Maxim Dounin <mdounin@mdounin.ru>
parents:
7081
diff
changeset
|
640 ngx_slab_free_pages(pool, page, size); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
641 |
2711
6b87e875e87a
fix building by BCC without NGX_DEBUG_MALLOC
Igor Sysoev <igor@sysoev.ru>
parents:
2611
diff
changeset
|
642 ngx_slab_junk(p, size << ngx_pagesize_shift); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
643 |
2711
6b87e875e87a
fix building by BCC without NGX_DEBUG_MALLOC
Igor Sysoev <igor@sysoev.ru>
parents:
2611
diff
changeset
|
644 return; |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
645 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
646 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
647 /* not reached */ |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
648 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
649 return; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
650 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
651 done: |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
652 |
6828 | 653 pool->stats[slot].used--; |
654 | |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
655 ngx_slab_junk(p, size); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
656 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
657 return; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
658 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
659 wrong_chunk: |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
660 |
2611
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
661 ngx_slab_error(pool, NGX_LOG_ALERT, |
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
662 "ngx_slab_free(): pointer to wrong chunk"); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
663 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
664 goto fail; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
665 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
666 chunk_already_free: |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
667 |
2611
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
668 ngx_slab_error(pool, NGX_LOG_ALERT, |
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
669 "ngx_slab_free(): chunk is already free"); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
670 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
671 fail: |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
672 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
673 return; |
357
e260514b9ad4
nginx-0.0.7-2004-06-16-23:36:07 import
Igor Sysoev <igor@sysoev.ru>
parents:
diff
changeset
|
674 } |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
675 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
676 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
677 static ngx_slab_page_t * |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
678 ngx_slab_alloc_pages(ngx_slab_pool_t *pool, ngx_uint_t pages) |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
679 { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
680 ngx_slab_page_t *page, *p; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
681 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
682 for (page = pool->free.next; page != &pool->free; page = page->next) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
683 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
684 if (page->slab >= pages) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
685 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
686 if (page->slab > pages) { |
5718
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
687 page[page->slab - 1].prev = (uintptr_t) &page[pages]; |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
688 |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
689 page[pages].slab = page->slab - pages; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
690 page[pages].next = page->next; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
691 page[pages].prev = page->prev; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
692 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
693 p = (ngx_slab_page_t *) page->prev; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
694 p->next = &page[pages]; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
695 page->next->prev = (uintptr_t) &page[pages]; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
696 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
697 } else { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
698 p = (ngx_slab_page_t *) page->prev; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
699 p->next = page->next; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
700 page->next->prev = page->prev; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
701 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
702 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
703 page->slab = pages | NGX_SLAB_PAGE_START; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
704 page->next = NULL; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
705 page->prev = NGX_SLAB_PAGE; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
706 |
6829
6e757036e588
Slab: free pages statistics.
Ruslan Ermilov <ru@nginx.com>
parents:
6828
diff
changeset
|
707 pool->pfree -= pages; |
6e757036e588
Slab: free pages statistics.
Ruslan Ermilov <ru@nginx.com>
parents:
6828
diff
changeset
|
708 |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
709 if (--pages == 0) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
710 return page; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
711 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
712 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
713 for (p = page + 1; pages; pages--) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
714 p->slab = NGX_SLAB_PAGE_BUSY; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
715 p->next = NULL; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
716 p->prev = NGX_SLAB_PAGE; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
717 p++; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
718 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
719 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
720 return page; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
721 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
722 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
723 |
5634
5024d29354f1
Core: slab log_nomem flag.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5362
diff
changeset
|
724 if (pool->log_nomem) { |
5024d29354f1
Core: slab log_nomem flag.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5362
diff
changeset
|
725 ngx_slab_error(pool, NGX_LOG_CRIT, |
5024d29354f1
Core: slab log_nomem flag.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5362
diff
changeset
|
726 "ngx_slab_alloc() failed: no memory"); |
5024d29354f1
Core: slab log_nomem flag.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5362
diff
changeset
|
727 } |
966
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
728 |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
729 return NULL; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
730 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
731 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
732 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
733 static void |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
734 ngx_slab_free_pages(ngx_slab_pool_t *pool, ngx_slab_page_t *page, |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
735 ngx_uint_t pages) |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
736 { |
5718
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
737 ngx_slab_page_t *prev, *join; |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
738 |
6829
6e757036e588
Slab: free pages statistics.
Ruslan Ermilov <ru@nginx.com>
parents:
6828
diff
changeset
|
739 pool->pfree += pages; |
6e757036e588
Slab: free pages statistics.
Ruslan Ermilov <ru@nginx.com>
parents:
6828
diff
changeset
|
740 |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
741 page->slab = pages--; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
742 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
743 if (pages) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
744 ngx_memzero(&page[1], pages * sizeof(ngx_slab_page_t)); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
745 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
746 |
966
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
747 if (page->next) { |
6823
88c8c3d65184
Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents:
6809
diff
changeset
|
748 prev = ngx_slab_page_prev(page); |
966
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
749 prev->next = page->next; |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
750 page->next->prev = page->prev; |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
751 } |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
752 |
5718
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
753 join = page + page->slab; |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
754 |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
755 if (join < pool->last) { |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
756 |
6823
88c8c3d65184
Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents:
6809
diff
changeset
|
757 if (ngx_slab_page_type(join) == NGX_SLAB_PAGE) { |
5718
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
758 |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
759 if (join->next != NULL) { |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
760 pages += join->slab; |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
761 page->slab += join->slab; |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
762 |
6823
88c8c3d65184
Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents:
6809
diff
changeset
|
763 prev = ngx_slab_page_prev(join); |
5718
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
764 prev->next = join->next; |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
765 join->next->prev = join->prev; |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
766 |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
767 join->slab = NGX_SLAB_PAGE_FREE; |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
768 join->next = NULL; |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
769 join->prev = NGX_SLAB_PAGE; |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
770 } |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
771 } |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
772 } |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
773 |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
774 if (page > pool->pages) { |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
775 join = page - 1; |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
776 |
6823
88c8c3d65184
Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents:
6809
diff
changeset
|
777 if (ngx_slab_page_type(join) == NGX_SLAB_PAGE) { |
5718
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
778 |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
779 if (join->slab == NGX_SLAB_PAGE_FREE) { |
6823
88c8c3d65184
Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents:
6809
diff
changeset
|
780 join = ngx_slab_page_prev(join); |
5718
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
781 } |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
782 |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
783 if (join->next != NULL) { |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
784 pages += join->slab; |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
785 join->slab += page->slab; |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
786 |
6823
88c8c3d65184
Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents:
6809
diff
changeset
|
787 prev = ngx_slab_page_prev(join); |
5718
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
788 prev->next = join->next; |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
789 join->next->prev = join->prev; |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
790 |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
791 page->slab = NGX_SLAB_PAGE_FREE; |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
792 page->next = NULL; |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
793 page->prev = NGX_SLAB_PAGE; |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
794 |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
795 page = join; |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
796 } |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
797 } |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
798 } |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
799 |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
800 if (pages) { |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
801 page[pages].prev = (uintptr_t) page; |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
802 } |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
803 |
966
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
804 page->prev = (uintptr_t) &pool->free; |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
805 page->next = pool->free.next; |
966
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
806 |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
807 page->next->prev = (uintptr_t) page; |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
808 |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
809 pool->free.next = page; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
810 } |
2611
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
811 |
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
812 |
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
813 static void |
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
814 ngx_slab_error(ngx_slab_pool_t *pool, ngx_uint_t level, char *text) |
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
815 { |
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
816 ngx_log_error(level, ngx_cycle->log, 0, "%s%s", text, pool->log_ctx); |
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
817 } |