]> git.proxmox.com Git - rustc.git/blob - src/jemalloc/include/jemalloc/internal/util.h
Imported Upstream version 1.9.0+dfsg1
[rustc.git] / src / jemalloc / include / jemalloc / internal / util.h
1 /******************************************************************************/
2 #ifdef JEMALLOC_H_TYPES
3
4 #ifdef _WIN32
5 # ifdef _WIN64
6 # define FMT64_PREFIX "ll"
7 # define FMTPTR_PREFIX "ll"
8 # else
9 # define FMT64_PREFIX "ll"
10 # define FMTPTR_PREFIX ""
11 # endif
12 # define FMTd32 "d"
13 # define FMTu32 "u"
14 # define FMTx32 "x"
15 # define FMTd64 FMT64_PREFIX "d"
16 # define FMTu64 FMT64_PREFIX "u"
17 # define FMTx64 FMT64_PREFIX "x"
18 # define FMTdPTR FMTPTR_PREFIX "d"
19 # define FMTuPTR FMTPTR_PREFIX "u"
20 # define FMTxPTR FMTPTR_PREFIX "x"
21 #else
22 # include <inttypes.h>
23 # define FMTd32 PRId32
24 # define FMTu32 PRIu32
25 # define FMTx32 PRIx32
26 # define FMTd64 PRId64
27 # define FMTu64 PRIu64
28 # define FMTx64 PRIx64
29 # define FMTdPTR PRIdPTR
30 # define FMTuPTR PRIuPTR
31 # define FMTxPTR PRIxPTR
32 #endif
33
34 /* Size of stack-allocated buffer passed to buferror(). */
35 #define BUFERROR_BUF 64
36
37 /*
38 * Size of stack-allocated buffer used by malloc_{,v,vc}printf(). This must be
39 * large enough for all possible uses within jemalloc.
40 */
41 #define MALLOC_PRINTF_BUFSIZE 4096
42
43 /*
44 * Wrap a cpp argument that contains commas such that it isn't broken up into
45 * multiple arguments.
46 */
47 #define JEMALLOC_ARG_CONCAT(...) __VA_ARGS__
48
49 /*
50 * Silence compiler warnings due to uninitialized values. This is used
51 * wherever the compiler fails to recognize that the variable is never used
52 * uninitialized.
53 */
54 #ifdef JEMALLOC_CC_SILENCE
55 # define JEMALLOC_CC_SILENCE_INIT(v) = v
56 #else
57 # define JEMALLOC_CC_SILENCE_INIT(v)
58 #endif
59
60 #define JEMALLOC_GNUC_PREREQ(major, minor) \
61 (!defined(__clang__) && \
62 (__GNUC__ > (major) || (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor))))
63 #ifndef __has_builtin
64 # define __has_builtin(builtin) (0)
65 #endif
66 #define JEMALLOC_CLANG_HAS_BUILTIN(builtin) \
67 (defined(__clang__) && __has_builtin(builtin))
68
69 #ifdef __GNUC__
70 # define likely(x) __builtin_expect(!!(x), 1)
71 # define unlikely(x) __builtin_expect(!!(x), 0)
72 # if JEMALLOC_GNUC_PREREQ(4, 6) || \
73 JEMALLOC_CLANG_HAS_BUILTIN(__builtin_unreachable)
74 # define unreachable() __builtin_unreachable()
75 # else
76 # define unreachable()
77 # endif
78 #else
79 # define likely(x) !!(x)
80 # define unlikely(x) !!(x)
81 # define unreachable()
82 #endif
83
84 #include "jemalloc/internal/assert.h"
85
86 /* Use to assert a particular configuration, e.g., cassert(config_debug). */
87 #define cassert(c) do { \
88 if (unlikely(!(c))) \
89 not_reached(); \
90 } while (0)
91
92 #endif /* JEMALLOC_H_TYPES */
93 /******************************************************************************/
94 #ifdef JEMALLOC_H_STRUCTS
95
96 #endif /* JEMALLOC_H_STRUCTS */
97 /******************************************************************************/
98 #ifdef JEMALLOC_H_EXTERNS
99
100 int buferror(int err, char *buf, size_t buflen);
101 uintmax_t malloc_strtoumax(const char *restrict nptr,
102 char **restrict endptr, int base);
103 void malloc_write(const char *s);
104
105 /*
106 * malloc_vsnprintf() supports a subset of snprintf(3) that avoids floating
107 * point math.
108 */
109 int malloc_vsnprintf(char *str, size_t size, const char *format,
110 va_list ap);
111 int malloc_snprintf(char *str, size_t size, const char *format, ...)
112 JEMALLOC_FORMAT_PRINTF(3, 4);
113 void malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque,
114 const char *format, va_list ap);
115 void malloc_cprintf(void (*write)(void *, const char *), void *cbopaque,
116 const char *format, ...) JEMALLOC_FORMAT_PRINTF(3, 4);
117 void malloc_printf(const char *format, ...) JEMALLOC_FORMAT_PRINTF(1, 2);
118
119 #endif /* JEMALLOC_H_EXTERNS */
120 /******************************************************************************/
121 #ifdef JEMALLOC_H_INLINES
122
123 #ifndef JEMALLOC_ENABLE_INLINE
124 unsigned ffs_llu(unsigned long long bitmap);
125 unsigned ffs_lu(unsigned long bitmap);
126 unsigned ffs_u(unsigned bitmap);
127 unsigned ffs_zu(size_t bitmap);
128 unsigned ffs_u64(uint64_t bitmap);
129 unsigned ffs_u32(uint32_t bitmap);
130 uint64_t pow2_ceil_u64(uint64_t x);
131 uint32_t pow2_ceil_u32(uint32_t x);
132 size_t pow2_ceil_zu(size_t x);
133 unsigned lg_floor(size_t x);
134 void set_errno(int errnum);
135 int get_errno(void);
136 #endif
137
138 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_UTIL_C_))
139
140 /* Sanity check. */
141 #if !defined(JEMALLOC_INTERNAL_FFSLL) || !defined(JEMALLOC_INTERNAL_FFSL) \
142 || !defined(JEMALLOC_INTERNAL_FFS)
143 # error JEMALLOC_INTERNAL_FFS{,L,LL} should have been defined by configure
144 #endif
145
146 JEMALLOC_ALWAYS_INLINE unsigned
147 ffs_llu(unsigned long long bitmap)
148 {
149
150 return (JEMALLOC_INTERNAL_FFSLL(bitmap));
151 }
152
153 JEMALLOC_ALWAYS_INLINE unsigned
154 ffs_lu(unsigned long bitmap)
155 {
156
157 return (JEMALLOC_INTERNAL_FFSL(bitmap));
158 }
159
160 JEMALLOC_ALWAYS_INLINE unsigned
161 ffs_u(unsigned bitmap)
162 {
163
164 return (JEMALLOC_INTERNAL_FFS(bitmap));
165 }
166
167 JEMALLOC_ALWAYS_INLINE unsigned
168 ffs_zu(size_t bitmap)
169 {
170
171 #if LG_SIZEOF_PTR == LG_SIZEOF_INT
172 return (ffs_u(bitmap));
173 #elif LG_SIZEOF_PTR == LG_SIZEOF_LONG
174 return (ffs_lu(bitmap));
175 #elif LG_SIZEOF_PTR == LG_SIZEOF_LONG_LONG
176 return (ffs_llu(bitmap));
177 #else
178 #error No implementation for size_t ffs()
179 #endif
180 }
181
182 JEMALLOC_ALWAYS_INLINE unsigned
183 ffs_u64(uint64_t bitmap)
184 {
185
186 #if LG_SIZEOF_LONG == 3
187 return (ffs_lu(bitmap));
188 #elif LG_SIZEOF_LONG_LONG == 3
189 return (ffs_llu(bitmap));
190 #else
191 #error No implementation for 64-bit ffs()
192 #endif
193 }
194
195 JEMALLOC_ALWAYS_INLINE unsigned
196 ffs_u32(uint32_t bitmap)
197 {
198
199 #if LG_SIZEOF_INT == 2
200 return (ffs_u(bitmap));
201 #else
202 #error No implementation for 32-bit ffs()
203 #endif
204 return (ffs_u(bitmap));
205 }
206
207 JEMALLOC_INLINE uint64_t
208 pow2_ceil_u64(uint64_t x)
209 {
210
211 x--;
212 x |= x >> 1;
213 x |= x >> 2;
214 x |= x >> 4;
215 x |= x >> 8;
216 x |= x >> 16;
217 x |= x >> 32;
218 x++;
219 return (x);
220 }
221
222 JEMALLOC_INLINE uint32_t
223 pow2_ceil_u32(uint32_t x)
224 {
225
226 x--;
227 x |= x >> 1;
228 x |= x >> 2;
229 x |= x >> 4;
230 x |= x >> 8;
231 x |= x >> 16;
232 x++;
233 return (x);
234 }
235
236 /* Compute the smallest power of 2 that is >= x. */
237 JEMALLOC_INLINE size_t
238 pow2_ceil_zu(size_t x)
239 {
240
241 #if (LG_SIZEOF_PTR == 3)
242 return (pow2_ceil_u64(x));
243 #else
244 return (pow2_ceil_u32(x));
245 #endif
246 }
247
248 #if (defined(__i386__) || defined(__amd64__) || defined(__x86_64__))
249 JEMALLOC_INLINE unsigned
250 lg_floor(size_t x)
251 {
252 size_t ret;
253
254 assert(x != 0);
255
256 asm ("bsr %1, %0"
257 : "=r"(ret) // Outputs.
258 : "r"(x) // Inputs.
259 );
260 assert(ret < UINT_MAX);
261 return ((unsigned)ret);
262 }
263 #elif (defined(_MSC_VER))
264 JEMALLOC_INLINE unsigned
265 lg_floor(size_t x)
266 {
267 unsigned long ret;
268
269 assert(x != 0);
270
271 #if (LG_SIZEOF_PTR == 3)
272 _BitScanReverse64(&ret, x);
273 #elif (LG_SIZEOF_PTR == 2)
274 _BitScanReverse(&ret, x);
275 #else
276 # error "Unsupported type size for lg_floor()"
277 #endif
278 assert(ret < UINT_MAX);
279 return ((unsigned)ret);
280 }
281 #elif (defined(JEMALLOC_HAVE_BUILTIN_CLZ))
282 JEMALLOC_INLINE unsigned
283 lg_floor(size_t x)
284 {
285
286 assert(x != 0);
287
288 #if (LG_SIZEOF_PTR == LG_SIZEOF_INT)
289 return (((8 << LG_SIZEOF_PTR) - 1) - __builtin_clz(x));
290 #elif (LG_SIZEOF_PTR == LG_SIZEOF_LONG)
291 return (((8 << LG_SIZEOF_PTR) - 1) - __builtin_clzl(x));
292 #else
293 # error "Unsupported type size for lg_floor()"
294 #endif
295 }
296 #else
297 JEMALLOC_INLINE unsigned
298 lg_floor(size_t x)
299 {
300
301 assert(x != 0);
302
303 x |= (x >> 1);
304 x |= (x >> 2);
305 x |= (x >> 4);
306 x |= (x >> 8);
307 x |= (x >> 16);
308 #if (LG_SIZEOF_PTR == 3)
309 x |= (x >> 32);
310 #endif
311 if (x == SIZE_T_MAX)
312 return ((8 << LG_SIZEOF_PTR) - 1);
313 x++;
314 return (ffs_zu(x) - 2);
315 }
316 #endif
317
318 /* Set error code. */
319 JEMALLOC_INLINE void
320 set_errno(int errnum)
321 {
322
323 #ifdef _WIN32
324 SetLastError(errnum);
325 #else
326 errno = errnum;
327 #endif
328 }
329
330 /* Get last error code. */
331 JEMALLOC_INLINE int
332 get_errno(void)
333 {
334
335 #ifdef _WIN32
336 return (GetLastError());
337 #else
338 return (errno);
339 #endif
340 }
341 #endif
342
343 #endif /* JEMALLOC_H_INLINES */
344 /******************************************************************************/