]>
Commit | Line | Data |
---|---|---|
970d7e83 LB |
1 | /******************************************************************************/ |
2 | #ifdef JEMALLOC_H_TYPES | |
3 | ||
54a0048b SL |
4 | #ifdef _WIN32 |
5 | # ifdef _WIN64 | |
6 | # define FMT64_PREFIX "ll" | |
7 | # define FMTPTR_PREFIX "ll" | |
8 | # else | |
9 | # define FMT64_PREFIX "ll" | |
10 | # define FMTPTR_PREFIX "" | |
11 | # endif | |
12 | # define FMTd32 "d" | |
13 | # define FMTu32 "u" | |
14 | # define FMTx32 "x" | |
15 | # define FMTd64 FMT64_PREFIX "d" | |
16 | # define FMTu64 FMT64_PREFIX "u" | |
17 | # define FMTx64 FMT64_PREFIX "x" | |
18 | # define FMTdPTR FMTPTR_PREFIX "d" | |
19 | # define FMTuPTR FMTPTR_PREFIX "u" | |
20 | # define FMTxPTR FMTPTR_PREFIX "x" | |
21 | #else | |
22 | # include <inttypes.h> | |
23 | # define FMTd32 PRId32 | |
24 | # define FMTu32 PRIu32 | |
25 | # define FMTx32 PRIx32 | |
26 | # define FMTd64 PRId64 | |
27 | # define FMTu64 PRIu64 | |
28 | # define FMTx64 PRIx64 | |
29 | # define FMTdPTR PRIdPTR | |
30 | # define FMTuPTR PRIuPTR | |
31 | # define FMTxPTR PRIxPTR | |
32 | #endif | |
33 | ||
970d7e83 LB |
34 | /* Size of stack-allocated buffer passed to buferror(). */ |
35 | #define BUFERROR_BUF 64 | |
36 | ||
37 | /* | |
38 | * Size of stack-allocated buffer used by malloc_{,v,vc}printf(). This must be | |
39 | * large enough for all possible uses within jemalloc. | |
40 | */ | |
41 | #define MALLOC_PRINTF_BUFSIZE 4096 | |
42 | ||
43 | /* | |
44 | * Wrap a cpp argument that contains commas such that it isn't broken up into | |
45 | * multiple arguments. | |
46 | */ | |
1a4d82fc | 47 | #define JEMALLOC_ARG_CONCAT(...) __VA_ARGS__ |
970d7e83 LB |
48 | |
49 | /* | |
50 | * Silence compiler warnings due to uninitialized values. This is used | |
51 | * wherever the compiler fails to recognize that the variable is never used | |
52 | * uninitialized. | |
53 | */ | |
54 | #ifdef JEMALLOC_CC_SILENCE | |
54a0048b | 55 | # define JEMALLOC_CC_SILENCE_INIT(v) = v |
1a4d82fc | 56 | #else |
54a0048b | 57 | # define JEMALLOC_CC_SILENCE_INIT(v) |
970d7e83 LB |
58 | #endif |
59 | ||
54a0048b SL |
60 | #define JEMALLOC_GNUC_PREREQ(major, minor) \ |
61 | (!defined(__clang__) && \ | |
62 | (__GNUC__ > (major) || (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))) | |
63 | #ifndef __has_builtin | |
64 | # define __has_builtin(builtin) (0) | |
970d7e83 | 65 | #endif |
54a0048b SL |
66 | #define JEMALLOC_CLANG_HAS_BUILTIN(builtin) \ |
67 | (defined(__clang__) && __has_builtin(builtin)) | |
970d7e83 | 68 | |
54a0048b SL |
69 | #ifdef __GNUC__ |
70 | # define likely(x) __builtin_expect(!!(x), 1) | |
71 | # define unlikely(x) __builtin_expect(!!(x), 0) | |
72 | # if JEMALLOC_GNUC_PREREQ(4, 6) || \ | |
73 | JEMALLOC_CLANG_HAS_BUILTIN(__builtin_unreachable) | |
74 | # define unreachable() __builtin_unreachable() | |
75 | # else | |
76 | # define unreachable() | |
77 | # endif | |
78 | #else | |
79 | # define likely(x) !!(x) | |
80 | # define unlikely(x) !!(x) | |
81 | # define unreachable() | |
970d7e83 LB |
82 | #endif |
83 | ||
54a0048b | 84 | #include "jemalloc/internal/assert.h" |
1a4d82fc JJ |
85 | |
86 | /* Use to assert a particular configuration, e.g., cassert(config_debug). */ | |
87 | #define cassert(c) do { \ | |
88 | if (unlikely(!(c))) \ | |
89 | not_reached(); \ | |
90 | } while (0) | |
970d7e83 LB |
91 | |
92 | #endif /* JEMALLOC_H_TYPES */ | |
93 | /******************************************************************************/ | |
94 | #ifdef JEMALLOC_H_STRUCTS | |
95 | ||
96 | #endif /* JEMALLOC_H_STRUCTS */ | |
97 | /******************************************************************************/ | |
98 | #ifdef JEMALLOC_H_EXTERNS | |
99 | ||
1a4d82fc JJ |
100 | int buferror(int err, char *buf, size_t buflen); |
101 | uintmax_t malloc_strtoumax(const char *restrict nptr, | |
102 | char **restrict endptr, int base); | |
970d7e83 LB |
103 | void malloc_write(const char *s); |
104 | ||
105 | /* | |
106 | * malloc_vsnprintf() supports a subset of snprintf(3) that avoids floating | |
107 | * point math. | |
108 | */ | |
109 | int malloc_vsnprintf(char *str, size_t size, const char *format, | |
110 | va_list ap); | |
111 | int malloc_snprintf(char *str, size_t size, const char *format, ...) | |
54a0048b | 112 | JEMALLOC_FORMAT_PRINTF(3, 4); |
970d7e83 LB |
113 | void malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque, |
114 | const char *format, va_list ap); | |
115 | void malloc_cprintf(void (*write)(void *, const char *), void *cbopaque, | |
54a0048b SL |
116 | const char *format, ...) JEMALLOC_FORMAT_PRINTF(3, 4); |
117 | void malloc_printf(const char *format, ...) JEMALLOC_FORMAT_PRINTF(1, 2); | |
970d7e83 LB |
118 | |
119 | #endif /* JEMALLOC_H_EXTERNS */ | |
120 | /******************************************************************************/ | |
121 | #ifdef JEMALLOC_H_INLINES | |
122 | ||
123 | #ifndef JEMALLOC_ENABLE_INLINE | |
54a0048b SL |
124 | unsigned ffs_llu(unsigned long long bitmap); |
125 | unsigned ffs_lu(unsigned long bitmap); | |
126 | unsigned ffs_u(unsigned bitmap); | |
127 | unsigned ffs_zu(size_t bitmap); | |
128 | unsigned ffs_u64(uint64_t bitmap); | |
129 | unsigned ffs_u32(uint32_t bitmap); | |
130 | uint64_t pow2_ceil_u64(uint64_t x); | |
131 | uint32_t pow2_ceil_u32(uint32_t x); | |
132 | size_t pow2_ceil_zu(size_t x); | |
133 | unsigned lg_floor(size_t x); | |
970d7e83 LB |
134 | void set_errno(int errnum); |
135 | int get_errno(void); | |
136 | #endif | |
137 | ||
138 | #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_UTIL_C_)) | |
1a4d82fc | 139 | |
54a0048b SL |
140 | /* Sanity check. */ |
141 | #if !defined(JEMALLOC_INTERNAL_FFSLL) || !defined(JEMALLOC_INTERNAL_FFSL) \ | |
142 | || !defined(JEMALLOC_INTERNAL_FFS) | |
143 | # error JEMALLOC_INTERNAL_FFS{,L,LL} should have been defined by configure | |
1a4d82fc JJ |
144 | #endif |
145 | ||
54a0048b SL |
146 | JEMALLOC_ALWAYS_INLINE unsigned |
147 | ffs_llu(unsigned long long bitmap) | |
1a4d82fc JJ |
148 | { |
149 | ||
54a0048b | 150 | return (JEMALLOC_INTERNAL_FFSLL(bitmap)); |
1a4d82fc JJ |
151 | } |
152 | ||
54a0048b SL |
153 | JEMALLOC_ALWAYS_INLINE unsigned |
154 | ffs_lu(unsigned long bitmap) | |
1a4d82fc JJ |
155 | { |
156 | ||
54a0048b | 157 | return (JEMALLOC_INTERNAL_FFSL(bitmap)); |
1a4d82fc JJ |
158 | } |
159 | ||
54a0048b SL |
160 | JEMALLOC_ALWAYS_INLINE unsigned |
161 | ffs_u(unsigned bitmap) | |
162 | { | |
163 | ||
164 | return (JEMALLOC_INTERNAL_FFS(bitmap)); | |
165 | } | |
166 | ||
167 | JEMALLOC_ALWAYS_INLINE unsigned | |
168 | ffs_zu(size_t bitmap) | |
169 | { | |
170 | ||
171 | #if LG_SIZEOF_PTR == LG_SIZEOF_INT | |
172 | return (ffs_u(bitmap)); | |
173 | #elif LG_SIZEOF_PTR == LG_SIZEOF_LONG | |
174 | return (ffs_lu(bitmap)); | |
175 | #elif LG_SIZEOF_PTR == LG_SIZEOF_LONG_LONG | |
176 | return (ffs_llu(bitmap)); | |
177 | #else | |
178 | #error No implementation for size_t ffs() | |
179 | #endif | |
180 | } | |
181 | ||
182 | JEMALLOC_ALWAYS_INLINE unsigned | |
183 | ffs_u64(uint64_t bitmap) | |
184 | { | |
185 | ||
186 | #if LG_SIZEOF_LONG == 3 | |
187 | return (ffs_lu(bitmap)); | |
188 | #elif LG_SIZEOF_LONG_LONG == 3 | |
189 | return (ffs_llu(bitmap)); | |
190 | #else | |
191 | #error No implementation for 64-bit ffs() | |
192 | #endif | |
193 | } | |
194 | ||
195 | JEMALLOC_ALWAYS_INLINE unsigned | |
196 | ffs_u32(uint32_t bitmap) | |
197 | { | |
198 | ||
199 | #if LG_SIZEOF_INT == 2 | |
200 | return (ffs_u(bitmap)); | |
201 | #else | |
202 | #error No implementation for 32-bit ffs() | |
203 | #endif | |
204 | return (ffs_u(bitmap)); | |
205 | } | |
206 | ||
207 | JEMALLOC_INLINE uint64_t | |
208 | pow2_ceil_u64(uint64_t x) | |
970d7e83 LB |
209 | { |
210 | ||
211 | x--; | |
212 | x |= x >> 1; | |
213 | x |= x >> 2; | |
214 | x |= x >> 4; | |
215 | x |= x >> 8; | |
216 | x |= x >> 16; | |
970d7e83 | 217 | x |= x >> 32; |
970d7e83 LB |
218 | x++; |
219 | return (x); | |
220 | } | |
221 | ||
54a0048b SL |
222 | JEMALLOC_INLINE uint32_t |
223 | pow2_ceil_u32(uint32_t x) | |
224 | { | |
225 | ||
226 | x--; | |
227 | x |= x >> 1; | |
228 | x |= x >> 2; | |
229 | x |= x >> 4; | |
230 | x |= x >> 8; | |
231 | x |= x >> 16; | |
232 | x++; | |
233 | return (x); | |
234 | } | |
235 | ||
236 | /* Compute the smallest power of 2 that is >= x. */ | |
1a4d82fc | 237 | JEMALLOC_INLINE size_t |
54a0048b SL |
238 | pow2_ceil_zu(size_t x) |
239 | { | |
240 | ||
241 | #if (LG_SIZEOF_PTR == 3) | |
242 | return (pow2_ceil_u64(x)); | |
243 | #else | |
244 | return (pow2_ceil_u32(x)); | |
245 | #endif | |
246 | } | |
247 | ||
248 | #if (defined(__i386__) || defined(__amd64__) || defined(__x86_64__)) | |
249 | JEMALLOC_INLINE unsigned | |
1a4d82fc JJ |
250 | lg_floor(size_t x) |
251 | { | |
252 | size_t ret; | |
253 | ||
54a0048b SL |
254 | assert(x != 0); |
255 | ||
1a4d82fc JJ |
256 | asm ("bsr %1, %0" |
257 | : "=r"(ret) // Outputs. | |
258 | : "r"(x) // Inputs. | |
259 | ); | |
54a0048b SL |
260 | assert(ret < UINT_MAX); |
261 | return ((unsigned)ret); | |
1a4d82fc JJ |
262 | } |
263 | #elif (defined(_MSC_VER)) | |
54a0048b | 264 | JEMALLOC_INLINE unsigned |
1a4d82fc JJ |
265 | lg_floor(size_t x) |
266 | { | |
54a0048b SL |
267 | unsigned long ret; |
268 | ||
269 | assert(x != 0); | |
1a4d82fc JJ |
270 | |
271 | #if (LG_SIZEOF_PTR == 3) | |
54a0048b | 272 | _BitScanReverse64(&ret, x); |
1a4d82fc | 273 | #elif (LG_SIZEOF_PTR == 2) |
54a0048b | 274 | _BitScanReverse(&ret, x); |
1a4d82fc | 275 | #else |
54a0048b | 276 | # error "Unsupported type size for lg_floor()" |
1a4d82fc | 277 | #endif |
54a0048b SL |
278 | assert(ret < UINT_MAX); |
279 | return ((unsigned)ret); | |
1a4d82fc JJ |
280 | } |
281 | #elif (defined(JEMALLOC_HAVE_BUILTIN_CLZ)) | |
54a0048b | 282 | JEMALLOC_INLINE unsigned |
1a4d82fc JJ |
283 | lg_floor(size_t x) |
284 | { | |
285 | ||
54a0048b SL |
286 | assert(x != 0); |
287 | ||
1a4d82fc JJ |
288 | #if (LG_SIZEOF_PTR == LG_SIZEOF_INT) |
289 | return (((8 << LG_SIZEOF_PTR) - 1) - __builtin_clz(x)); | |
290 | #elif (LG_SIZEOF_PTR == LG_SIZEOF_LONG) | |
291 | return (((8 << LG_SIZEOF_PTR) - 1) - __builtin_clzl(x)); | |
292 | #else | |
54a0048b | 293 | # error "Unsupported type size for lg_floor()" |
1a4d82fc JJ |
294 | #endif |
295 | } | |
296 | #else | |
54a0048b | 297 | JEMALLOC_INLINE unsigned |
1a4d82fc JJ |
298 | lg_floor(size_t x) |
299 | { | |
300 | ||
54a0048b SL |
301 | assert(x != 0); |
302 | ||
1a4d82fc JJ |
303 | x |= (x >> 1); |
304 | x |= (x >> 2); | |
305 | x |= (x >> 4); | |
306 | x |= (x >> 8); | |
307 | x |= (x >> 16); | |
54a0048b | 308 | #if (LG_SIZEOF_PTR == 3) |
1a4d82fc | 309 | x |= (x >> 32); |
1a4d82fc | 310 | #endif |
54a0048b SL |
311 | if (x == SIZE_T_MAX) |
312 | return ((8 << LG_SIZEOF_PTR) - 1); | |
313 | x++; | |
314 | return (ffs_zu(x) - 2); | |
1a4d82fc JJ |
315 | } |
316 | #endif | |
317 | ||
54a0048b | 318 | /* Set error code. */ |
970d7e83 LB |
319 | JEMALLOC_INLINE void |
320 | set_errno(int errnum) | |
321 | { | |
322 | ||
323 | #ifdef _WIN32 | |
324 | SetLastError(errnum); | |
325 | #else | |
326 | errno = errnum; | |
327 | #endif | |
328 | } | |
329 | ||
54a0048b | 330 | /* Get last error code. */ |
970d7e83 LB |
331 | JEMALLOC_INLINE int |
332 | get_errno(void) | |
333 | { | |
334 | ||
335 | #ifdef _WIN32 | |
336 | return (GetLastError()); | |
337 | #else | |
338 | return (errno); | |
339 | #endif | |
340 | } | |
341 | #endif | |
342 | ||
343 | #endif /* JEMALLOC_H_INLINES */ | |
344 | /******************************************************************************/ |