src/jemalloc/include/jemalloc/internal/util.h

   1 /******************************************************************************/
   2 #ifdef JEMALLOC_H_TYPES
   3
   4 #ifdef _WIN32
   5 #  ifdef _WIN64
   6 #    define FMT64_PREFIX "ll"
   7 #    define FMTPTR_PREFIX "ll"
   8 #  else
   9 #    define FMT64_PREFIX "ll"
  10 #    define FMTPTR_PREFIX ""
  11 #  endif
  12 #  define FMTd32 "d"
  13 #  define FMTu32 "u"
  14 #  define FMTx32 "x"
  15 #  define FMTd64 FMT64_PREFIX "d"
  16 #  define FMTu64 FMT64_PREFIX "u"
  17 #  define FMTx64 FMT64_PREFIX "x"
  18 #  define FMTdPTR FMTPTR_PREFIX "d"
  19 #  define FMTuPTR FMTPTR_PREFIX "u"
  20 #  define FMTxPTR FMTPTR_PREFIX "x"
  21 #else
  22 #  include <inttypes.h>
  23 #  define FMTd32 PRId32
  24 #  define FMTu32 PRIu32
  25 #  define FMTx32 PRIx32
  26 #  define FMTd64 PRId64
  27 #  define FMTu64 PRIu64
  28 #  define FMTx64 PRIx64
  29 #  define FMTdPTR PRIdPTR
  30 #  define FMTuPTR PRIuPTR
  31 #  define FMTxPTR PRIxPTR
  32 #endif
  33
  34 /* Size of stack-allocated buffer passed to buferror(). */
  35 #define BUFERROR_BUF            64
  36
  37 /*
  38  * Size of stack-allocated buffer used by malloc_{,v,vc}printf().  This must be
  39  * large enough for all possible uses within jemalloc.
  40  */
  41 #define MALLOC_PRINTF_BUFSIZE   4096
  42
  43 /*
  44  * Wrap a cpp argument that contains commas such that it isn't broken up into
  45  * multiple arguments.
  46  */
  47 #define JEMALLOC_ARG_CONCAT(...) __VA_ARGS__
  48
  49 /*
  50  * Silence compiler warnings due to uninitialized values.  This is used
  51  * wherever the compiler fails to recognize that the variable is never used
  52  * uninitialized.
  53  */
  54 #ifdef JEMALLOC_CC_SILENCE
  55 #       define JEMALLOC_CC_SILENCE_INIT(v) = v
  56 #else
  57 #       define JEMALLOC_CC_SILENCE_INIT(v)
  58 #endif
  59
  60 #define JEMALLOC_GNUC_PREREQ(major, minor)                              \
  61     (!defined(__clang__) &&                                             \
  62     (__GNUC__ > (major) || (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor))))
  63 #ifndef __has_builtin
  64 #  define __has_builtin(builtin) (0)
  65 #endif
  66 #define JEMALLOC_CLANG_HAS_BUILTIN(builtin)                             \
  67     (defined(__clang__) && __has_builtin(builtin))
  68
  69 #ifdef __GNUC__
  70 #       define likely(x)   __builtin_expect(!!(x), 1)
  71 #       define unlikely(x) __builtin_expect(!!(x), 0)
  72 #  if JEMALLOC_GNUC_PREREQ(4, 6) ||                                     \
  73       JEMALLOC_CLANG_HAS_BUILTIN(__builtin_unreachable)
  74 #       define unreachable() __builtin_unreachable()
  75 #  else
  76 #       define unreachable()
  77 #  endif
  78 #else
  79 #       define likely(x)   !!(x)
  80 #       define unlikely(x) !!(x)
  81 #       define unreachable()
  82 #endif
  83
  84 #include "jemalloc/internal/assert.h"
  85
  86 /* Use to assert a particular configuration, e.g., cassert(config_debug). */
  87 #define cassert(c) do {                                                 \
  88         if (unlikely(!(c)))                                             \
  89                 not_reached();                                          \
  90 } while (0)
  91
  92 #endif /* JEMALLOC_H_TYPES */
  93 /******************************************************************************/
  94 #ifdef JEMALLOC_H_STRUCTS
  95
  96 #endif /* JEMALLOC_H_STRUCTS */
  97 /******************************************************************************/
  98 #ifdef JEMALLOC_H_EXTERNS
  99
 100 int     buferror(int err, char *buf, size_t buflen);
 101 uintmax_t       malloc_strtoumax(const char *restrict nptr,
 102     char **restrict endptr, int base);
 103 void    malloc_write(const char *s);
 104
 105 /*
 106  * malloc_vsnprintf() supports a subset of snprintf(3) that avoids floating
 107  * point math.
 108  */
 109 int     malloc_vsnprintf(char *str, size_t size, const char *format,
 110     va_list ap);
 111 int     malloc_snprintf(char *str, size_t size, const char *format, ...)
 112     JEMALLOC_FORMAT_PRINTF(3, 4);
 113 void    malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque,
 114     const char *format, va_list ap);
 115 void malloc_cprintf(void (*write)(void *, const char *), void *cbopaque,
 116     const char *format, ...) JEMALLOC_FORMAT_PRINTF(3, 4);
 117 void    malloc_printf(const char *format, ...) JEMALLOC_FORMAT_PRINTF(1, 2);
 118
 119 #endif /* JEMALLOC_H_EXTERNS */
 120 /******************************************************************************/
 121 #ifdef JEMALLOC_H_INLINES
 122
 123 #ifndef JEMALLOC_ENABLE_INLINE
 124 unsigned        ffs_llu(unsigned long long bitmap);
 125 unsigned        ffs_lu(unsigned long bitmap);
 126 unsigned        ffs_u(unsigned bitmap);
 127 unsigned        ffs_zu(size_t bitmap);
 128 unsigned        ffs_u64(uint64_t bitmap);
 129 unsigned        ffs_u32(uint32_t bitmap);
 130 uint64_t        pow2_ceil_u64(uint64_t x);
 131 uint32_t        pow2_ceil_u32(uint32_t x);
 132 size_t  pow2_ceil_zu(size_t x);
 133 unsigned        lg_floor(size_t x);
 134 void    set_errno(int errnum);
 135 int     get_errno(void);
 136 #endif
 137
 138 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_UTIL_C_))
 139
 140 /* Sanity check. */
 141 #if !defined(JEMALLOC_INTERNAL_FFSLL) || !defined(JEMALLOC_INTERNAL_FFSL) \
 142     || !defined(JEMALLOC_INTERNAL_FFS)
 143 #  error JEMALLOC_INTERNAL_FFS{,L,LL} should have been defined by configure
 144 #endif
 145
 146 JEMALLOC_ALWAYS_INLINE unsigned
 147 ffs_llu(unsigned long long bitmap)
 148 {
 149
 150         return (JEMALLOC_INTERNAL_FFSLL(bitmap));
 151 }
 152
 153 JEMALLOC_ALWAYS_INLINE unsigned
 154 ffs_lu(unsigned long bitmap)
 155 {
 156
 157         return (JEMALLOC_INTERNAL_FFSL(bitmap));
 158 }
 159
 160 JEMALLOC_ALWAYS_INLINE unsigned
 161 ffs_u(unsigned bitmap)
 162 {
 163
 164         return (JEMALLOC_INTERNAL_FFS(bitmap));
 165 }
 166
 167 JEMALLOC_ALWAYS_INLINE unsigned
 168 ffs_zu(size_t bitmap)
 169 {
 170
 171 #if LG_SIZEOF_PTR == LG_SIZEOF_INT
 172         return (ffs_u(bitmap));
 173 #elif LG_SIZEOF_PTR == LG_SIZEOF_LONG
 174         return (ffs_lu(bitmap));
 175 #elif LG_SIZEOF_PTR == LG_SIZEOF_LONG_LONG
 176         return (ffs_llu(bitmap));
 177 #else
 178 #error No implementation for size_t ffs()
 179 #endif
 180 }
 181
 182 JEMALLOC_ALWAYS_INLINE unsigned
 183 ffs_u64(uint64_t bitmap)
 184 {
 185
 186 #if LG_SIZEOF_LONG == 3
 187         return (ffs_lu(bitmap));
 188 #elif LG_SIZEOF_LONG_LONG == 3
 189         return (ffs_llu(bitmap));
 190 #else
 191 #error No implementation for 64-bit ffs()
 192 #endif
 193 }
 194
 195 JEMALLOC_ALWAYS_INLINE unsigned
 196 ffs_u32(uint32_t bitmap)
 197 {
 198
 199 #if LG_SIZEOF_INT == 2
 200         return (ffs_u(bitmap));
 201 #else
 202 #error No implementation for 32-bit ffs()
 203 #endif
 204         return (ffs_u(bitmap));
 205 }
 206
 207 JEMALLOC_INLINE uint64_t
 208 pow2_ceil_u64(uint64_t x)
 209 {
 210
 211         x--;
 212         x |= x >> 1;
 213         x |= x >> 2;
 214         x |= x >> 4;
 215         x |= x >> 8;
 216         x |= x >> 16;
 217         x |= x >> 32;
 218         x++;
 219         return (x);
 220 }
 221
 222 JEMALLOC_INLINE uint32_t
 223 pow2_ceil_u32(uint32_t x)
 224 {
 225
 226         x--;
 227         x |= x >> 1;
 228         x |= x >> 2;
 229         x |= x >> 4;
 230         x |= x >> 8;
 231         x |= x >> 16;
 232         x++;
 233         return (x);
 234 }
 235
 236 /* Compute the smallest power of 2 that is >= x. */
 237 JEMALLOC_INLINE size_t
 238 pow2_ceil_zu(size_t x)
 239 {
 240
 241 #if (LG_SIZEOF_PTR == 3)
 242         return (pow2_ceil_u64(x));
 243 #else
 244         return (pow2_ceil_u32(x));
 245 #endif
 246 }
 247
 248 #if (defined(__i386__) || defined(__amd64__) || defined(__x86_64__))
 249 JEMALLOC_INLINE unsigned
 250 lg_floor(size_t x)
 251 {
 252         size_t ret;
 253
 254         assert(x != 0);
 255
 256         asm ("bsr %1, %0"
 257             : "=r"(ret) // Outputs.
 258             : "r"(x)    // Inputs.
 259             );
 260         assert(ret < UINT_MAX);
 261         return ((unsigned)ret);
 262 }
 263 #elif (defined(_MSC_VER))
 264 JEMALLOC_INLINE unsigned
 265 lg_floor(size_t x)
 266 {
 267         unsigned long ret;
 268
 269         assert(x != 0);
 270
 271 #if (LG_SIZEOF_PTR == 3)
 272         _BitScanReverse64(&ret, x);
 273 #elif (LG_SIZEOF_PTR == 2)
 274         _BitScanReverse(&ret, x);
 275 #else
 276 #  error "Unsupported type size for lg_floor()"
 277 #endif
 278         assert(ret < UINT_MAX);
 279         return ((unsigned)ret);
 280 }
 281 #elif (defined(JEMALLOC_HAVE_BUILTIN_CLZ))
 282 JEMALLOC_INLINE unsigned
 283 lg_floor(size_t x)
 284 {
 285
 286         assert(x != 0);
 287
 288 #if (LG_SIZEOF_PTR == LG_SIZEOF_INT)
 289         return (((8 << LG_SIZEOF_PTR) - 1) - __builtin_clz(x));
 290 #elif (LG_SIZEOF_PTR == LG_SIZEOF_LONG)
 291         return (((8 << LG_SIZEOF_PTR) - 1) - __builtin_clzl(x));
 292 #else
 293 #  error "Unsupported type size for lg_floor()"
 294 #endif
 295 }
 296 #else
 297 JEMALLOC_INLINE unsigned
 298 lg_floor(size_t x)
 299 {
 300
 301         assert(x != 0);
 302
 303         x |= (x >> 1);
 304         x |= (x >> 2);
 305         x |= (x >> 4);
 306         x |= (x >> 8);
 307         x |= (x >> 16);
 308 #if (LG_SIZEOF_PTR == 3)
 309         x |= (x >> 32);
 310 #endif
 311         if (x == SIZE_T_MAX)
 312                 return ((8 << LG_SIZEOF_PTR) - 1);
 313         x++;
 314         return (ffs_zu(x) - 2);
 315 }
 316 #endif
 317
 318 /* Set error code. */
 319 JEMALLOC_INLINE void
 320 set_errno(int errnum)
 321 {
 322
 323 #ifdef _WIN32
 324         SetLastError(errnum);
 325 #else
 326         errno = errnum;
 327 #endif
 328 }
 329
 330 /* Get last error code. */
 331 JEMALLOC_INLINE int
 332 get_errno(void)
 333 {
 334
 335 #ifdef _WIN32
 336         return (GetLastError());
 337 #else
 338         return (errno);
 339 #endif
 340 }
 341 #endif
 342
 343 #endif /* JEMALLOC_H_INLINES */
 344 /******************************************************************************/