src/jemalloc/src/zone.c

   1 #include "jemalloc/internal/jemalloc_internal.h"
   2 #ifndef JEMALLOC_ZONE
   3 #  error "This source file is for zones on Darwin (OS X)."
   4 #endif
   5
   6 /*
   7  * The malloc_default_purgeable_zone function is only available on >= 10.6.
   8  * We need to check whether it is present at runtime, thus the weak_import.
   9  */
  10 extern malloc_zone_t *malloc_default_purgeable_zone(void)
  11 JEMALLOC_ATTR(weak_import);
  12
  13 /******************************************************************************/
  14 /* Data. */
  15
  16 static malloc_zone_t zone;
  17 static struct malloc_introspection_t zone_introspect;
  18
  19 /******************************************************************************/
  20 /* Function prototypes for non-inline static functions. */
  21
  22 static size_t   zone_size(malloc_zone_t *zone, void *ptr);
  23 static void     *zone_malloc(malloc_zone_t *zone, size_t size);
  24 static void     *zone_calloc(malloc_zone_t *zone, size_t num, size_t size);
  25 static void     *zone_valloc(malloc_zone_t *zone, size_t size);
  26 static void     zone_free(malloc_zone_t *zone, void *ptr);
  27 static void     *zone_realloc(malloc_zone_t *zone, void *ptr, size_t size);
  28 #if (JEMALLOC_ZONE_VERSION >= 5)
  29 static void     *zone_memalign(malloc_zone_t *zone, size_t alignment,
  30 #endif
  31 #if (JEMALLOC_ZONE_VERSION >= 6)
  32     size_t size);
  33 static void     zone_free_definite_size(malloc_zone_t *zone, void *ptr,
  34     size_t size);
  35 #endif
  36 static void     *zone_destroy(malloc_zone_t *zone);
  37 static size_t   zone_good_size(malloc_zone_t *zone, size_t size);
  38 static void     zone_force_lock(malloc_zone_t *zone);
  39 static void     zone_force_unlock(malloc_zone_t *zone);
  40
  41 /******************************************************************************/
  42 /*
  43  * Functions.
  44  */
  45
  46 static size_t
  47 zone_size(malloc_zone_t *zone, void *ptr)
  48 {
  49
  50         /*
  51          * There appear to be places within Darwin (such as setenv(3)) that
  52          * cause calls to this function with pointers that *no* zone owns.  If
  53          * we knew that all pointers were owned by *some* zone, we could split
  54          * our zone into two parts, and use one as the default allocator and
  55          * the other as the default deallocator/reallocator.  Since that will
  56          * not work in practice, we must check all pointers to assure that they
  57          * reside within a mapped chunk before determining size.
  58          */
  59         return (ivsalloc(ptr, config_prof));
  60 }
  61
  62 static void *
  63 zone_malloc(malloc_zone_t *zone, size_t size)
  64 {
  65
  66         return (je_malloc(size));
  67 }
  68
  69 static void *
  70 zone_calloc(malloc_zone_t *zone, size_t num, size_t size)
  71 {
  72
  73         return (je_calloc(num, size));
  74 }
  75
  76 static void *
  77 zone_valloc(malloc_zone_t *zone, size_t size)
  78 {
  79         void *ret = NULL; /* Assignment avoids useless compiler warning. */
  80
  81         je_posix_memalign(&ret, PAGE, size);
  82
  83         return (ret);
  84 }
  85
  86 static void
  87 zone_free(malloc_zone_t *zone, void *ptr)
  88 {
  89
  90         if (ivsalloc(ptr, config_prof) != 0) {
  91                 je_free(ptr);
  92                 return;
  93         }
  94
  95         free(ptr);
  96 }
  97
  98 static void *
  99 zone_realloc(malloc_zone_t *zone, void *ptr, size_t size)
 100 {
 101
 102         if (ivsalloc(ptr, config_prof) != 0)
 103                 return (je_realloc(ptr, size));
 104
 105         return (realloc(ptr, size));
 106 }
 107
 108 #if (JEMALLOC_ZONE_VERSION >= 5)
 109 static void *
 110 zone_memalign(malloc_zone_t *zone, size_t alignment, size_t size)
 111 {
 112         void *ret = NULL; /* Assignment avoids useless compiler warning. */
 113
 114         je_posix_memalign(&ret, alignment, size);
 115
 116         return (ret);
 117 }
 118 #endif
 119
 120 #if (JEMALLOC_ZONE_VERSION >= 6)
 121 static void
 122 zone_free_definite_size(malloc_zone_t *zone, void *ptr, size_t size)
 123 {
 124         size_t alloc_size;
 125
 126         alloc_size = ivsalloc(ptr, config_prof);
 127         if (alloc_size != 0) {
 128                 assert(alloc_size == size);
 129                 je_free(ptr);
 130                 return;
 131         }
 132
 133         free(ptr);
 134 }
 135 #endif
 136
 137 static void *
 138 zone_destroy(malloc_zone_t *zone)
 139 {
 140
 141         /* This function should never be called. */
 142         not_reached();
 143         return (NULL);
 144 }
 145
 146 static size_t
 147 zone_good_size(malloc_zone_t *zone, size_t size)
 148 {
 149
 150         if (size == 0)
 151                 size = 1;
 152         return (s2u(size));
 153 }
 154
 155 static void
 156 zone_force_lock(malloc_zone_t *zone)
 157 {
 158
 159         if (isthreaded)
 160                 jemalloc_prefork();
 161 }
 162
 163 static void
 164 zone_force_unlock(malloc_zone_t *zone)
 165 {
 166
 167         if (isthreaded)
 168                 jemalloc_postfork_parent();
 169 }
 170
 171 JEMALLOC_ATTR(constructor)
 172 void
 173 register_zone(void)
 174 {
 175
 176         /*
 177          * If something else replaced the system default zone allocator, don't
 178          * register jemalloc's.
 179          */
 180         malloc_zone_t *default_zone = malloc_default_zone();
 181         malloc_zone_t *purgeable_zone = NULL;
 182         if (!default_zone->zone_name ||
 183             strcmp(default_zone->zone_name, "DefaultMallocZone") != 0) {
 184                 return;
 185         }
 186
 187         zone.size = (void *)zone_size;
 188         zone.malloc = (void *)zone_malloc;
 189         zone.calloc = (void *)zone_calloc;
 190         zone.valloc = (void *)zone_valloc;
 191         zone.free = (void *)zone_free;
 192         zone.realloc = (void *)zone_realloc;
 193         zone.destroy = (void *)zone_destroy;
 194         zone.zone_name = "jemalloc_zone";
 195         zone.batch_malloc = NULL;
 196         zone.batch_free = NULL;
 197         zone.introspect = &zone_introspect;
 198         zone.version = JEMALLOC_ZONE_VERSION;
 199 #if (JEMALLOC_ZONE_VERSION >= 5)
 200         zone.memalign = zone_memalign;
 201 #endif
 202 #if (JEMALLOC_ZONE_VERSION >= 6)
 203         zone.free_definite_size = zone_free_definite_size;
 204 #endif
 205 #if (JEMALLOC_ZONE_VERSION >= 8)
 206         zone.pressure_relief = NULL;
 207 #endif
 208
 209         zone_introspect.enumerator = NULL;
 210         zone_introspect.good_size = (void *)zone_good_size;
 211         zone_introspect.check = NULL;
 212         zone_introspect.print = NULL;
 213         zone_introspect.log = NULL;
 214         zone_introspect.force_lock = (void *)zone_force_lock;
 215         zone_introspect.force_unlock = (void *)zone_force_unlock;
 216         zone_introspect.statistics = NULL;
 217 #if (JEMALLOC_ZONE_VERSION >= 6)
 218         zone_introspect.zone_locked = NULL;
 219 #endif
 220 #if (JEMALLOC_ZONE_VERSION >= 7)
 221         zone_introspect.enable_discharge_checking = NULL;
 222         zone_introspect.disable_discharge_checking = NULL;
 223         zone_introspect.discharge = NULL;
 224 #ifdef __BLOCKS__
 225         zone_introspect.enumerate_discharged_pointers = NULL;
 226 #else
 227         zone_introspect.enumerate_unavailable_without_blocks = NULL;
 228 #endif
 229 #endif
 230
 231         /*
 232          * The default purgeable zone is created lazily by OSX's libc.  It uses
 233          * the default zone when it is created for "small" allocations
 234          * (< 15 KiB), but assumes the default zone is a scalable_zone.  This
 235          * obviously fails when the default zone is the jemalloc zone, so
 236          * malloc_default_purgeable_zone is called beforehand so that the
 237          * default purgeable zone is created when the default zone is still
 238          * a scalable_zone.  As purgeable zones only exist on >= 10.6, we need
 239          * to check for the existence of malloc_default_purgeable_zone() at
 240          * run time.
 241          */
 242         if (malloc_default_purgeable_zone != NULL)
 243                 purgeable_zone = malloc_default_purgeable_zone();
 244
 245         /* Register the custom zone.  At this point it won't be the default. */
 246         malloc_zone_register(&zone);
 247
 248         do {
 249                 default_zone = malloc_default_zone();
 250                 /*
 251                  * Unregister and reregister the default zone.  On OSX >= 10.6,
 252                  * unregistering takes the last registered zone and places it
 253                  * at the location of the specified zone.  Unregistering the
 254                  * default zone thus makes the last registered one the default.
 255                  * On OSX < 10.6, unregistering shifts all registered zones.
 256                  * The first registered zone then becomes the default.
 257                  */
 258                 malloc_zone_unregister(default_zone);
 259                 malloc_zone_register(default_zone);
 260                 /*
 261                  * On OSX 10.6, having the default purgeable zone appear before
 262                  * the default zone makes some things crash because it thinks it
 263                  * owns the default zone allocated pointers.  We thus
 264                  * unregister/re-register it in order to ensure it's always
 265                  * after the default zone.  On OSX < 10.6, there is no purgeable
 266                  * zone, so this does nothing.  On OSX >= 10.6, unregistering
 267                  * replaces the purgeable zone with the last registered zone
 268                  * above, i.e. the default zone.  Registering it again then puts
 269                  * it at the end, obviously after the default zone.
 270                  */
 271                 if (purgeable_zone) {
 272                         malloc_zone_unregister(purgeable_zone);
 273                         malloc_zone_register(purgeable_zone);
 274                 }
 275         } while (malloc_default_zone() != &zone);
 276 }