]>
Commit | Line | Data |
---|---|---|
34dc7c2f BB |
1 | /* |
2 | * CDDL HEADER START | |
3 | * | |
4 | * The contents of this file are subject to the terms of the | |
5 | * Common Development and Distribution License, Version 1.0 only | |
6 | * (the "License"). You may not use this file except in compliance | |
7 | * with the License. | |
8 | * | |
9 | * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE | |
10 | * or http://www.opensolaris.org/os/licensing. | |
11 | * See the License for the specific language governing permissions | |
12 | * and limitations under the License. | |
13 | * | |
14 | * When distributing Covered Code, include this CDDL HEADER in each | |
15 | * file and include the License file at usr/src/OPENSOLARIS.LICENSE. | |
16 | * If applicable, add the following below this CDDL HEADER, with the | |
17 | * fields enclosed by brackets "[]" replaced with your own identifying | |
18 | * information: Portions Copyright [yyyy] [name of copyright owner] | |
19 | * | |
20 | * CDDL HEADER END | |
21 | */ | |
22 | /* | |
23 | * Copyright 2004 Sun Microsystems, Inc. All rights reserved. | |
24 | * Use is subject to license terms. | |
25 | */ | |
26 | /* | |
27 | * Portions Copyright 2006 OmniTI, Inc. | |
28 | */ | |
29 | ||
30 | #ifndef _UMEM_IMPL_H | |
31 | #define _UMEM_IMPL_H | |
32 | ||
33 | /* #pragma ident "@(#)umem_impl.h 1.6 05/06/08 SMI" */ | |
34 | ||
35 | #include <umem.h> | |
36 | ||
37 | #ifdef HAVE_SYS_SYSMACROS_H | |
38 | #include <sys/sysmacros.h> | |
39 | #endif | |
40 | ||
41 | #if HAVE_SYS_TIME_H | |
42 | #include <sys/time.h> | |
43 | #endif | |
44 | ||
45 | #include <sys/vmem.h> | |
46 | #ifdef HAVE_THREAD_H | |
47 | # include <thread.h> | |
48 | #else | |
49 | # include "sol_compat.h" | |
50 | #endif | |
51 | ||
52 | #ifdef __cplusplus | |
53 | extern "C" { | |
54 | #endif | |
55 | ||
56 | /* | |
57 | * umem memory allocator: implementation-private data structures | |
58 | */ | |
59 | ||
60 | /* | |
61 | * Internal flags for umem_cache_create | |
62 | */ | |
63 | #define UMC_QCACHE 0x00100000 | |
64 | #define UMC_INTERNAL 0x80000000 | |
65 | ||
66 | /* | |
67 | * Cache flags | |
68 | */ | |
69 | #define UMF_AUDIT 0x00000001 /* transaction auditing */ | |
70 | #define UMF_DEADBEEF 0x00000002 /* deadbeef checking */ | |
71 | #define UMF_REDZONE 0x00000004 /* redzone checking */ | |
72 | #define UMF_CONTENTS 0x00000008 /* freed-buffer content logging */ | |
73 | #define UMF_CHECKSIGNAL 0x00000010 /* abort when in signal context */ | |
74 | #define UMF_NOMAGAZINE 0x00000020 /* disable per-cpu magazines */ | |
75 | #define UMF_FIREWALL 0x00000040 /* put all bufs before unmapped pages */ | |
76 | #define UMF_LITE 0x00000100 /* lightweight debugging */ | |
77 | ||
78 | #define UMF_HASH 0x00000200 /* cache has hash table */ | |
79 | #define UMF_RANDOMIZE 0x00000400 /* randomize other umem_flags */ | |
80 | ||
81 | #define UMF_BUFTAG (UMF_DEADBEEF | UMF_REDZONE) | |
82 | #define UMF_TOUCH (UMF_BUFTAG | UMF_LITE | UMF_CONTENTS) | |
83 | #define UMF_RANDOM (UMF_TOUCH | UMF_AUDIT | UMF_NOMAGAZINE) | |
84 | #define UMF_DEBUG (UMF_RANDOM | UMF_FIREWALL) | |
85 | ||
86 | #define UMEM_STACK_DEPTH umem_stack_depth | |
87 | ||
88 | #define UMEM_FREE_PATTERN 0xdeadbeefdeadbeefULL | |
89 | #define UMEM_UNINITIALIZED_PATTERN 0xbaddcafebaddcafeULL | |
90 | #define UMEM_REDZONE_PATTERN 0xfeedfacefeedfaceULL | |
91 | #define UMEM_REDZONE_BYTE 0xbb | |
92 | ||
93 | #define UMEM_FATAL_FLAGS (UMEM_NOFAIL) | |
94 | #define UMEM_SLEEP_FLAGS (0) | |
95 | ||
96 | /* | |
97 | * Redzone size encodings for umem_alloc() / umem_free(). We encode the | |
98 | * allocation size, rather than storing it directly, so that umem_free() | |
99 | * can distinguish frees of the wrong size from redzone violations. | |
100 | */ | |
101 | #define UMEM_SIZE_ENCODE(x) (251 * (x) + 1) | |
102 | #define UMEM_SIZE_DECODE(x) ((x) / 251) | |
103 | #define UMEM_SIZE_VALID(x) ((x) % 251 == 1) | |
104 | ||
105 | /* | |
106 | * The bufctl (buffer control) structure keeps some minimal information | |
107 | * about each buffer: its address, its slab, and its current linkage, | |
108 | * which is either on the slab's freelist (if the buffer is free), or | |
109 | * on the cache's buf-to-bufctl hash table (if the buffer is allocated). | |
110 | * In the case of non-hashed, or "raw", caches (the common case), only | |
111 | * the freelist linkage is necessary: the buffer address is at a fixed | |
112 | * offset from the bufctl address, and the slab is at the end of the page. | |
113 | * | |
114 | * NOTE: bc_next must be the first field; raw buffers have linkage only. | |
115 | */ | |
116 | typedef struct umem_bufctl { | |
117 | struct umem_bufctl *bc_next; /* next bufctl struct */ | |
118 | void *bc_addr; /* address of buffer */ | |
119 | struct umem_slab *bc_slab; /* controlling slab */ | |
120 | } umem_bufctl_t; | |
121 | ||
122 | /* | |
123 | * The UMF_AUDIT version of the bufctl structure. The beginning of this | |
124 | * structure must be identical to the normal bufctl structure so that | |
125 | * pointers are interchangeable. | |
126 | */ | |
127 | ||
128 | #define UMEM_BUFCTL_AUDIT_SIZE_DEPTH(frames) \ | |
129 | ((size_t)(&((umem_bufctl_audit_t *)0)->bc_stack[frames])) | |
130 | ||
131 | /* | |
132 | * umem_bufctl_audits must be allocated from a UMC_NOHASH cache, so we | |
133 | * require that 2 of them, plus 2 buftags, plus a umem_slab_t, all fit on | |
134 | * a single page. | |
135 | * | |
136 | * For ILP32, this is about 1000 frames. | |
137 | * For LP64, this is about 490 frames. | |
138 | */ | |
139 | ||
140 | #define UMEM_BUFCTL_AUDIT_ALIGN 32 | |
141 | ||
142 | #define UMEM_BUFCTL_AUDIT_MAX_SIZE \ | |
143 | (P2ALIGN((PAGESIZE - sizeof (umem_slab_t))/2 - \ | |
144 | sizeof (umem_buftag_t), UMEM_BUFCTL_AUDIT_ALIGN)) | |
145 | ||
146 | #define UMEM_MAX_STACK_DEPTH \ | |
147 | ((UMEM_BUFCTL_AUDIT_MAX_SIZE - \ | |
148 | UMEM_BUFCTL_AUDIT_SIZE_DEPTH(0)) / sizeof (uintptr_t)) | |
149 | ||
150 | typedef struct umem_bufctl_audit { | |
151 | struct umem_bufctl *bc_next; /* next bufctl struct */ | |
152 | void *bc_addr; /* address of buffer */ | |
153 | struct umem_slab *bc_slab; /* controlling slab */ | |
154 | umem_cache_t *bc_cache; /* controlling cache */ | |
155 | hrtime_t bc_timestamp; /* transaction time */ | |
156 | thread_t bc_thread; /* thread doing transaction */ | |
157 | struct umem_bufctl *bc_lastlog; /* last log entry */ | |
158 | void *bc_contents; /* contents at last free */ | |
159 | int bc_depth; /* stack depth */ | |
160 | uintptr_t bc_stack[1]; /* pc stack */ | |
161 | } umem_bufctl_audit_t; | |
162 | ||
163 | #define UMEM_LOCAL_BUFCTL_AUDIT(bcpp) \ | |
164 | *(bcpp) = (umem_bufctl_audit_t *) \ | |
165 | alloca(UMEM_BUFCTL_AUDIT_SIZE) | |
166 | ||
167 | #define UMEM_BUFCTL_AUDIT_SIZE \ | |
168 | UMEM_BUFCTL_AUDIT_SIZE_DEPTH(UMEM_STACK_DEPTH) | |
169 | ||
170 | /* | |
171 | * A umem_buftag structure is appended to each buffer whenever any of the | |
172 | * UMF_BUFTAG flags (UMF_DEADBEEF, UMF_REDZONE, UMF_VERIFY) are set. | |
173 | */ | |
174 | typedef struct umem_buftag { | |
175 | uint64_t bt_redzone; /* 64-bit redzone pattern */ | |
176 | umem_bufctl_t *bt_bufctl; /* bufctl */ | |
177 | intptr_t bt_bxstat; /* bufctl ^ (alloc/free) */ | |
178 | } umem_buftag_t; | |
179 | ||
180 | #define UMEM_BUFTAG(cp, buf) \ | |
181 | ((umem_buftag_t *)((char *)(buf) + (cp)->cache_buftag)) | |
182 | ||
183 | #define UMEM_BUFCTL(cp, buf) \ | |
184 | ((umem_bufctl_t *)((char *)(buf) + (cp)->cache_bufctl)) | |
185 | ||
186 | #define UMEM_BUF(cp, bcp) \ | |
187 | ((void *)((char *)(bcp) - (cp)->cache_bufctl)) | |
188 | ||
189 | #define UMEM_SLAB(cp, buf) \ | |
190 | ((umem_slab_t *)P2END((uintptr_t)(buf), (cp)->cache_slabsize) - 1) | |
191 | ||
192 | #define UMEM_CPU_CACHE(cp, cpu) \ | |
193 | (umem_cpu_cache_t *)((char *)cp + cpu->cpu_cache_offset) | |
194 | ||
195 | #define UMEM_MAGAZINE_VALID(cp, mp) \ | |
196 | (((umem_slab_t *)P2END((uintptr_t)(mp), PAGESIZE) - 1)->slab_cache == \ | |
197 | (cp)->cache_magtype->mt_cache) | |
198 | ||
199 | #define UMEM_SLAB_MEMBER(sp, buf) \ | |
200 | ((size_t)(buf) - (size_t)(sp)->slab_base < \ | |
201 | (sp)->slab_cache->cache_slabsize) | |
202 | ||
203 | #define UMEM_BUFTAG_ALLOC 0xa110c8edUL | |
204 | #define UMEM_BUFTAG_FREE 0xf4eef4eeUL | |
205 | ||
206 | typedef struct umem_slab { | |
207 | struct umem_cache *slab_cache; /* controlling cache */ | |
208 | void *slab_base; /* base of allocated memory */ | |
209 | struct umem_slab *slab_next; /* next slab on freelist */ | |
210 | struct umem_slab *slab_prev; /* prev slab on freelist */ | |
211 | struct umem_bufctl *slab_head; /* first free buffer */ | |
212 | long slab_refcnt; /* outstanding allocations */ | |
213 | long slab_chunks; /* chunks (bufs) in this slab */ | |
214 | } umem_slab_t; | |
215 | ||
216 | #define UMEM_HASH_INITIAL 64 | |
217 | ||
218 | #define UMEM_HASH(cp, buf) \ | |
219 | ((cp)->cache_hash_table + \ | |
220 | (((uintptr_t)(buf) >> (cp)->cache_hash_shift) & (cp)->cache_hash_mask)) | |
221 | ||
222 | typedef struct umem_magazine { | |
223 | void *mag_next; | |
224 | void *mag_round[1]; /* one or more rounds */ | |
225 | } umem_magazine_t; | |
226 | ||
227 | /* | |
228 | * The magazine types for fast per-cpu allocation | |
229 | */ | |
230 | typedef struct umem_magtype { | |
231 | int mt_magsize; /* magazine size (number of rounds) */ | |
232 | int mt_align; /* magazine alignment */ | |
233 | size_t mt_minbuf; /* all smaller buffers qualify */ | |
234 | size_t mt_maxbuf; /* no larger buffers qualify */ | |
235 | umem_cache_t *mt_cache; /* magazine cache */ | |
236 | } umem_magtype_t; | |
237 | ||
238 | #if (defined(__PTHREAD_MUTEX_SIZE__) && __PTHREAD_MUTEX_SIZE__ >= 24) || defined(UMEM_PTHREAD_MUTEX_TOO_BIG) | |
239 | #define UMEM_CPU_CACHE_SIZE 128 /* must be power of 2 */ | |
240 | #else | |
241 | #define UMEM_CPU_CACHE_SIZE 64 /* must be power of 2 */ | |
242 | #endif | |
243 | #define UMEM_CPU_PAD (UMEM_CPU_CACHE_SIZE - sizeof (mutex_t) - \ | |
244 | 2 * sizeof (uint_t) - 2 * sizeof (void *) - 4 * sizeof (int)) | |
245 | #define UMEM_CACHE_SIZE(ncpus) \ | |
246 | ((size_t)(&((umem_cache_t *)0)->cache_cpu[ncpus])) | |
247 | ||
248 | typedef struct umem_cpu_cache { | |
249 | mutex_t cc_lock; /* protects this cpu's local cache */ | |
250 | uint_t cc_alloc; /* allocations from this cpu */ | |
251 | uint_t cc_free; /* frees to this cpu */ | |
252 | umem_magazine_t *cc_loaded; /* the currently loaded magazine */ | |
253 | umem_magazine_t *cc_ploaded; /* the previously loaded magazine */ | |
254 | int cc_rounds; /* number of objects in loaded mag */ | |
255 | int cc_prounds; /* number of objects in previous mag */ | |
256 | int cc_magsize; /* number of rounds in a full mag */ | |
257 | int cc_flags; /* CPU-local copy of cache_flags */ | |
258 | #if (!defined(_LP64) || defined(UMEM_PTHREAD_MUTEX_TOO_BIG)) && !defined(_WIN32) | |
259 | /* on win32, UMEM_CPU_PAD evaluates to zero, and the MS compiler | |
260 | * won't allow static initialization of arrays containing structures | |
261 | * that contain zero size arrays */ | |
262 | char cc_pad[UMEM_CPU_PAD]; /* for nice alignment (32-bit) */ | |
263 | #endif | |
264 | } umem_cpu_cache_t; | |
265 | ||
266 | /* | |
267 | * The magazine lists used in the depot. | |
268 | */ | |
269 | typedef struct umem_maglist { | |
270 | umem_magazine_t *ml_list; /* magazine list */ | |
271 | long ml_total; /* number of magazines */ | |
272 | long ml_min; /* min since last update */ | |
273 | long ml_reaplimit; /* max reapable magazines */ | |
274 | uint64_t ml_alloc; /* allocations from this list */ | |
275 | } umem_maglist_t; | |
276 | ||
277 | #define UMEM_CACHE_NAMELEN 31 | |
278 | ||
279 | struct umem_cache { | |
280 | /* | |
281 | * Statistics | |
282 | */ | |
283 | uint64_t cache_slab_create; /* slab creates */ | |
284 | uint64_t cache_slab_destroy; /* slab destroys */ | |
285 | uint64_t cache_slab_alloc; /* slab layer allocations */ | |
286 | uint64_t cache_slab_free; /* slab layer frees */ | |
287 | uint64_t cache_alloc_fail; /* total failed allocations */ | |
288 | uint64_t cache_buftotal; /* total buffers */ | |
289 | uint64_t cache_bufmax; /* max buffers ever */ | |
290 | uint64_t cache_rescale; /* # of hash table rescales */ | |
291 | uint64_t cache_lookup_depth; /* hash lookup depth */ | |
292 | uint64_t cache_depot_contention; /* mutex contention count */ | |
293 | uint64_t cache_depot_contention_prev; /* previous snapshot */ | |
294 | ||
295 | /* | |
296 | * Cache properties | |
297 | */ | |
298 | char cache_name[UMEM_CACHE_NAMELEN + 1]; | |
299 | size_t cache_bufsize; /* object size */ | |
300 | size_t cache_align; /* object alignment */ | |
301 | umem_constructor_t *cache_constructor; | |
302 | umem_destructor_t *cache_destructor; | |
303 | umem_reclaim_t *cache_reclaim; | |
304 | void *cache_private; /* opaque arg to callbacks */ | |
305 | vmem_t *cache_arena; /* vmem source for slabs */ | |
306 | int cache_cflags; /* cache creation flags */ | |
307 | int cache_flags; /* various cache state info */ | |
308 | int cache_uflags; /* UMU_* flags */ | |
309 | uint32_t cache_mtbf; /* induced alloc failure rate */ | |
310 | umem_cache_t *cache_next; /* forward cache linkage */ | |
311 | umem_cache_t *cache_prev; /* backward cache linkage */ | |
312 | umem_cache_t *cache_unext; /* next in update list */ | |
313 | umem_cache_t *cache_uprev; /* prev in update list */ | |
314 | uint32_t cache_cpu_mask; /* mask for cpu offset */ | |
315 | ||
316 | /* | |
317 | * Slab layer | |
318 | */ | |
319 | mutex_t cache_lock; /* protects slab layer */ | |
320 | size_t cache_chunksize; /* buf + alignment [+ debug] */ | |
321 | size_t cache_slabsize; /* size of a slab */ | |
322 | size_t cache_bufctl; /* buf-to-bufctl distance */ | |
323 | size_t cache_buftag; /* buf-to-buftag distance */ | |
324 | size_t cache_verify; /* bytes to verify */ | |
325 | size_t cache_contents; /* bytes of saved content */ | |
326 | size_t cache_color; /* next slab color */ | |
327 | size_t cache_mincolor; /* maximum slab color */ | |
328 | size_t cache_maxcolor; /* maximum slab color */ | |
329 | size_t cache_hash_shift; /* get to interesting bits */ | |
330 | size_t cache_hash_mask; /* hash table mask */ | |
331 | umem_slab_t *cache_freelist; /* slab free list */ | |
332 | umem_slab_t cache_nullslab; /* end of freelist marker */ | |
333 | umem_cache_t *cache_bufctl_cache; /* source of bufctls */ | |
334 | umem_bufctl_t **cache_hash_table; /* hash table base */ | |
335 | /* | |
336 | * Depot layer | |
337 | */ | |
338 | mutex_t cache_depot_lock; /* protects depot */ | |
339 | umem_magtype_t *cache_magtype; /* magazine type */ | |
340 | umem_maglist_t cache_full; /* full magazines */ | |
341 | umem_maglist_t cache_empty; /* empty magazines */ | |
342 | ||
343 | /* | |
344 | * Per-CPU layer | |
345 | */ | |
346 | umem_cpu_cache_t cache_cpu[1]; /* cache_cpu_mask + 1 entries */ | |
347 | }; | |
348 | ||
349 | typedef struct umem_cpu_log_header { | |
350 | mutex_t clh_lock; | |
351 | char *clh_current; | |
352 | size_t clh_avail; | |
353 | int clh_chunk; | |
354 | int clh_hits; | |
355 | char clh_pad[UMEM_CPU_CACHE_SIZE - | |
356 | sizeof (mutex_t) - sizeof (char *) - | |
357 | sizeof (size_t) - 2 * sizeof (int)]; | |
358 | } umem_cpu_log_header_t; | |
359 | ||
360 | typedef struct umem_log_header { | |
361 | mutex_t lh_lock; | |
362 | char *lh_base; | |
363 | int *lh_free; | |
364 | size_t lh_chunksize; | |
365 | int lh_nchunks; | |
366 | int lh_head; | |
367 | int lh_tail; | |
368 | int lh_hits; | |
369 | umem_cpu_log_header_t lh_cpu[1]; /* actually umem_max_ncpus */ | |
370 | } umem_log_header_t; | |
371 | ||
372 | typedef struct umem_cpu { | |
373 | uint32_t cpu_cache_offset; | |
374 | uint32_t cpu_number; | |
375 | } umem_cpu_t; | |
376 | ||
377 | #define UMEM_MAXBUF 16384 | |
378 | ||
379 | #define UMEM_ALIGN 8 /* min guaranteed alignment */ | |
380 | #define UMEM_ALIGN_SHIFT 3 /* log2(UMEM_ALIGN) */ | |
381 | #define UMEM_VOID_FRACTION 8 /* never waste more than 1/8 of slab */ | |
382 | ||
383 | /* | |
384 | * For 64 bits, buffers >= 16 bytes must be 16-byte aligned | |
385 | */ | |
386 | #ifdef _LP64 | |
387 | #define UMEM_SECOND_ALIGN 16 | |
388 | #else | |
389 | #define UMEM_SECOND_ALIGN UMEM_ALIGN | |
390 | #endif | |
391 | ||
392 | #define MALLOC_MAGIC 0x3a10c000 /* 8-byte tag */ | |
393 | #define MEMALIGN_MAGIC 0x3e3a1000 | |
394 | ||
395 | #ifdef _LP64 | |
396 | #define MALLOC_SECOND_MAGIC 0x16ba7000 /* 8-byte tag, 16-aligned */ | |
397 | #define MALLOC_OVERSIZE_MAGIC 0x06e47000 /* 16-byte tag, _LP64 */ | |
398 | #endif | |
399 | ||
400 | #define UMEM_MALLOC_ENCODE(type, sz) (uint32_t)((type) - (sz)) | |
401 | #define UMEM_MALLOC_DECODE(stat, sz) (uint32_t)((stat) + (sz)) | |
402 | #define UMEM_FREE_PATTERN_32 (uint32_t)(UMEM_FREE_PATTERN) | |
403 | ||
404 | #define UMU_MAGAZINE_RESIZE 0x00000001 | |
405 | #define UMU_HASH_RESCALE 0x00000002 | |
406 | #define UMU_REAP 0x00000004 | |
407 | #define UMU_NOTIFY 0x08000000 | |
408 | #define UMU_ACTIVE 0x80000000 | |
409 | ||
410 | #define UMEM_READY_INIT_FAILED -1 | |
411 | #define UMEM_READY_STARTUP 1 | |
412 | #define UMEM_READY_INITING 2 | |
413 | #define UMEM_READY 3 | |
414 | ||
415 | #ifdef UMEM_STANDALONE | |
416 | extern void umem_startup(caddr_t, size_t, size_t, caddr_t, caddr_t); | |
417 | extern int umem_add(caddr_t, size_t); | |
418 | #endif | |
419 | ||
420 | #ifdef __cplusplus | |
421 | } | |
422 | #endif | |
423 | ||
424 | #endif /* _UMEM_IMPL_H */ |