]> git.proxmox.com Git - rustc.git/blame - src/jemalloc/include/jemalloc/internal/util.h
Imported Upstream version 1.9.0+dfsg1
[rustc.git] / src / jemalloc / include / jemalloc / internal / util.h
CommitLineData
970d7e83
LB
1/******************************************************************************/
2#ifdef JEMALLOC_H_TYPES
3
54a0048b
SL
4#ifdef _WIN32
5# ifdef _WIN64
6# define FMT64_PREFIX "ll"
7# define FMTPTR_PREFIX "ll"
8# else
9# define FMT64_PREFIX "ll"
10# define FMTPTR_PREFIX ""
11# endif
12# define FMTd32 "d"
13# define FMTu32 "u"
14# define FMTx32 "x"
15# define FMTd64 FMT64_PREFIX "d"
16# define FMTu64 FMT64_PREFIX "u"
17# define FMTx64 FMT64_PREFIX "x"
18# define FMTdPTR FMTPTR_PREFIX "d"
19# define FMTuPTR FMTPTR_PREFIX "u"
20# define FMTxPTR FMTPTR_PREFIX "x"
21#else
22# include <inttypes.h>
23# define FMTd32 PRId32
24# define FMTu32 PRIu32
25# define FMTx32 PRIx32
26# define FMTd64 PRId64
27# define FMTu64 PRIu64
28# define FMTx64 PRIx64
29# define FMTdPTR PRIdPTR
30# define FMTuPTR PRIuPTR
31# define FMTxPTR PRIxPTR
32#endif
33
970d7e83
LB
34/* Size of stack-allocated buffer passed to buferror(). */
35#define BUFERROR_BUF 64
36
37/*
38 * Size of stack-allocated buffer used by malloc_{,v,vc}printf(). This must be
39 * large enough for all possible uses within jemalloc.
40 */
41#define MALLOC_PRINTF_BUFSIZE 4096
42
43/*
44 * Wrap a cpp argument that contains commas such that it isn't broken up into
45 * multiple arguments.
46 */
1a4d82fc 47#define JEMALLOC_ARG_CONCAT(...) __VA_ARGS__
970d7e83
LB
48
49/*
50 * Silence compiler warnings due to uninitialized values. This is used
51 * wherever the compiler fails to recognize that the variable is never used
52 * uninitialized.
53 */
54#ifdef JEMALLOC_CC_SILENCE
54a0048b 55# define JEMALLOC_CC_SILENCE_INIT(v) = v
1a4d82fc 56#else
54a0048b 57# define JEMALLOC_CC_SILENCE_INIT(v)
970d7e83
LB
58#endif
59
54a0048b
SL
60#define JEMALLOC_GNUC_PREREQ(major, minor) \
61 (!defined(__clang__) && \
62 (__GNUC__ > (major) || (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor))))
63#ifndef __has_builtin
64# define __has_builtin(builtin) (0)
970d7e83 65#endif
54a0048b
SL
66#define JEMALLOC_CLANG_HAS_BUILTIN(builtin) \
67 (defined(__clang__) && __has_builtin(builtin))
970d7e83 68
54a0048b
SL
69#ifdef __GNUC__
70# define likely(x) __builtin_expect(!!(x), 1)
71# define unlikely(x) __builtin_expect(!!(x), 0)
72# if JEMALLOC_GNUC_PREREQ(4, 6) || \
73 JEMALLOC_CLANG_HAS_BUILTIN(__builtin_unreachable)
74# define unreachable() __builtin_unreachable()
75# else
76# define unreachable()
77# endif
78#else
79# define likely(x) !!(x)
80# define unlikely(x) !!(x)
81# define unreachable()
970d7e83
LB
82#endif
83
54a0048b 84#include "jemalloc/internal/assert.h"
1a4d82fc
JJ
85
86/* Use to assert a particular configuration, e.g., cassert(config_debug). */
87#define cassert(c) do { \
88 if (unlikely(!(c))) \
89 not_reached(); \
90} while (0)
970d7e83
LB
91
92#endif /* JEMALLOC_H_TYPES */
93/******************************************************************************/
94#ifdef JEMALLOC_H_STRUCTS
95
96#endif /* JEMALLOC_H_STRUCTS */
97/******************************************************************************/
98#ifdef JEMALLOC_H_EXTERNS
99
1a4d82fc
JJ
100int buferror(int err, char *buf, size_t buflen);
101uintmax_t malloc_strtoumax(const char *restrict nptr,
102 char **restrict endptr, int base);
970d7e83
LB
103void malloc_write(const char *s);
104
105/*
106 * malloc_vsnprintf() supports a subset of snprintf(3) that avoids floating
107 * point math.
108 */
109int malloc_vsnprintf(char *str, size_t size, const char *format,
110 va_list ap);
111int malloc_snprintf(char *str, size_t size, const char *format, ...)
54a0048b 112 JEMALLOC_FORMAT_PRINTF(3, 4);
970d7e83
LB
113void malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque,
114 const char *format, va_list ap);
115void malloc_cprintf(void (*write)(void *, const char *), void *cbopaque,
54a0048b
SL
116 const char *format, ...) JEMALLOC_FORMAT_PRINTF(3, 4);
117void malloc_printf(const char *format, ...) JEMALLOC_FORMAT_PRINTF(1, 2);
970d7e83
LB
118
119#endif /* JEMALLOC_H_EXTERNS */
120/******************************************************************************/
121#ifdef JEMALLOC_H_INLINES
122
123#ifndef JEMALLOC_ENABLE_INLINE
54a0048b
SL
124unsigned ffs_llu(unsigned long long bitmap);
125unsigned ffs_lu(unsigned long bitmap);
126unsigned ffs_u(unsigned bitmap);
127unsigned ffs_zu(size_t bitmap);
128unsigned ffs_u64(uint64_t bitmap);
129unsigned ffs_u32(uint32_t bitmap);
130uint64_t pow2_ceil_u64(uint64_t x);
131uint32_t pow2_ceil_u32(uint32_t x);
132size_t pow2_ceil_zu(size_t x);
133unsigned lg_floor(size_t x);
970d7e83
LB
134void set_errno(int errnum);
135int get_errno(void);
136#endif
137
138#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_UTIL_C_))
1a4d82fc 139
54a0048b
SL
140/* Sanity check. */
141#if !defined(JEMALLOC_INTERNAL_FFSLL) || !defined(JEMALLOC_INTERNAL_FFSL) \
142 || !defined(JEMALLOC_INTERNAL_FFS)
143# error JEMALLOC_INTERNAL_FFS{,L,LL} should have been defined by configure
1a4d82fc
JJ
144#endif
145
54a0048b
SL
146JEMALLOC_ALWAYS_INLINE unsigned
147ffs_llu(unsigned long long bitmap)
1a4d82fc
JJ
148{
149
54a0048b 150 return (JEMALLOC_INTERNAL_FFSLL(bitmap));
1a4d82fc
JJ
151}
152
54a0048b
SL
153JEMALLOC_ALWAYS_INLINE unsigned
154ffs_lu(unsigned long bitmap)
1a4d82fc
JJ
155{
156
54a0048b 157 return (JEMALLOC_INTERNAL_FFSL(bitmap));
1a4d82fc
JJ
158}
159
54a0048b
SL
160JEMALLOC_ALWAYS_INLINE unsigned
161ffs_u(unsigned bitmap)
162{
163
164 return (JEMALLOC_INTERNAL_FFS(bitmap));
165}
166
167JEMALLOC_ALWAYS_INLINE unsigned
168ffs_zu(size_t bitmap)
169{
170
171#if LG_SIZEOF_PTR == LG_SIZEOF_INT
172 return (ffs_u(bitmap));
173#elif LG_SIZEOF_PTR == LG_SIZEOF_LONG
174 return (ffs_lu(bitmap));
175#elif LG_SIZEOF_PTR == LG_SIZEOF_LONG_LONG
176 return (ffs_llu(bitmap));
177#else
178#error No implementation for size_t ffs()
179#endif
180}
181
182JEMALLOC_ALWAYS_INLINE unsigned
183ffs_u64(uint64_t bitmap)
184{
185
186#if LG_SIZEOF_LONG == 3
187 return (ffs_lu(bitmap));
188#elif LG_SIZEOF_LONG_LONG == 3
189 return (ffs_llu(bitmap));
190#else
191#error No implementation for 64-bit ffs()
192#endif
193}
194
195JEMALLOC_ALWAYS_INLINE unsigned
196ffs_u32(uint32_t bitmap)
197{
198
199#if LG_SIZEOF_INT == 2
200 return (ffs_u(bitmap));
201#else
202#error No implementation for 32-bit ffs()
203#endif
204 return (ffs_u(bitmap));
205}
206
207JEMALLOC_INLINE uint64_t
208pow2_ceil_u64(uint64_t x)
970d7e83
LB
209{
210
211 x--;
212 x |= x >> 1;
213 x |= x >> 2;
214 x |= x >> 4;
215 x |= x >> 8;
216 x |= x >> 16;
970d7e83 217 x |= x >> 32;
970d7e83
LB
218 x++;
219 return (x);
220}
221
54a0048b
SL
222JEMALLOC_INLINE uint32_t
223pow2_ceil_u32(uint32_t x)
224{
225
226 x--;
227 x |= x >> 1;
228 x |= x >> 2;
229 x |= x >> 4;
230 x |= x >> 8;
231 x |= x >> 16;
232 x++;
233 return (x);
234}
235
236/* Compute the smallest power of 2 that is >= x. */
1a4d82fc 237JEMALLOC_INLINE size_t
54a0048b
SL
238pow2_ceil_zu(size_t x)
239{
240
241#if (LG_SIZEOF_PTR == 3)
242 return (pow2_ceil_u64(x));
243#else
244 return (pow2_ceil_u32(x));
245#endif
246}
247
248#if (defined(__i386__) || defined(__amd64__) || defined(__x86_64__))
249JEMALLOC_INLINE unsigned
1a4d82fc
JJ
250lg_floor(size_t x)
251{
252 size_t ret;
253
54a0048b
SL
254 assert(x != 0);
255
1a4d82fc
JJ
256 asm ("bsr %1, %0"
257 : "=r"(ret) // Outputs.
258 : "r"(x) // Inputs.
259 );
54a0048b
SL
260 assert(ret < UINT_MAX);
261 return ((unsigned)ret);
1a4d82fc
JJ
262}
263#elif (defined(_MSC_VER))
54a0048b 264JEMALLOC_INLINE unsigned
1a4d82fc
JJ
265lg_floor(size_t x)
266{
54a0048b
SL
267 unsigned long ret;
268
269 assert(x != 0);
1a4d82fc
JJ
270
271#if (LG_SIZEOF_PTR == 3)
54a0048b 272 _BitScanReverse64(&ret, x);
1a4d82fc 273#elif (LG_SIZEOF_PTR == 2)
54a0048b 274 _BitScanReverse(&ret, x);
1a4d82fc 275#else
54a0048b 276# error "Unsupported type size for lg_floor()"
1a4d82fc 277#endif
54a0048b
SL
278 assert(ret < UINT_MAX);
279 return ((unsigned)ret);
1a4d82fc
JJ
280}
281#elif (defined(JEMALLOC_HAVE_BUILTIN_CLZ))
54a0048b 282JEMALLOC_INLINE unsigned
1a4d82fc
JJ
283lg_floor(size_t x)
284{
285
54a0048b
SL
286 assert(x != 0);
287
1a4d82fc
JJ
288#if (LG_SIZEOF_PTR == LG_SIZEOF_INT)
289 return (((8 << LG_SIZEOF_PTR) - 1) - __builtin_clz(x));
290#elif (LG_SIZEOF_PTR == LG_SIZEOF_LONG)
291 return (((8 << LG_SIZEOF_PTR) - 1) - __builtin_clzl(x));
292#else
54a0048b 293# error "Unsupported type size for lg_floor()"
1a4d82fc
JJ
294#endif
295}
296#else
54a0048b 297JEMALLOC_INLINE unsigned
1a4d82fc
JJ
298lg_floor(size_t x)
299{
300
54a0048b
SL
301 assert(x != 0);
302
1a4d82fc
JJ
303 x |= (x >> 1);
304 x |= (x >> 2);
305 x |= (x >> 4);
306 x |= (x >> 8);
307 x |= (x >> 16);
54a0048b 308#if (LG_SIZEOF_PTR == 3)
1a4d82fc 309 x |= (x >> 32);
1a4d82fc 310#endif
54a0048b
SL
311 if (x == SIZE_T_MAX)
312 return ((8 << LG_SIZEOF_PTR) - 1);
313 x++;
314 return (ffs_zu(x) - 2);
1a4d82fc
JJ
315}
316#endif
317
54a0048b 318/* Set error code. */
970d7e83
LB
319JEMALLOC_INLINE void
320set_errno(int errnum)
321{
322
323#ifdef _WIN32
324 SetLastError(errnum);
325#else
326 errno = errnum;
327#endif
328}
329
54a0048b 330/* Get last error code. */
970d7e83
LB
331JEMALLOC_INLINE int
332get_errno(void)
333{
334
335#ifdef _WIN32
336 return (GetLastError());
337#else
338 return (errno);
339#endif
340}
341#endif
342
343#endif /* JEMALLOC_H_INLINES */
344/******************************************************************************/