src/jemalloc/src/quarantine.c

   1 #define JEMALLOC_QUARANTINE_C_
   2 #include "jemalloc/internal/jemalloc_internal.h"
   3
   4 /*
   5  * Quarantine pointers close to NULL are used to encode state information that
   6  * is used for cleaning up during thread shutdown.
   7  */
   8 #define QUARANTINE_STATE_REINCARNATED   ((quarantine_t *)(uintptr_t)1)
   9 #define QUARANTINE_STATE_PURGATORY      ((quarantine_t *)(uintptr_t)2)
  10 #define QUARANTINE_STATE_MAX            QUARANTINE_STATE_PURGATORY
  11
  12 /******************************************************************************/
  13 /* Function prototypes for non-inline static functions. */
  14
  15 static quarantine_t     *quarantine_grow(tsd_t *tsd, quarantine_t *quarantine);
  16 static void     quarantine_drain_one(tsd_t *tsd, quarantine_t *quarantine);
  17 static void     quarantine_drain(tsd_t *tsd, quarantine_t *quarantine,
  18     size_t upper_bound);
  19
  20 /******************************************************************************/
  21
  22 static quarantine_t *
  23 quarantine_init(tsd_t *tsd, size_t lg_maxobjs)
  24 {
  25         quarantine_t *quarantine;
  26         size_t size;
  27
  28         assert(tsd_nominal(tsd));
  29
  30         size = offsetof(quarantine_t, objs) + ((ZU(1) << lg_maxobjs) *
  31             sizeof(quarantine_obj_t));
  32         quarantine = (quarantine_t *)iallocztm(tsd, size, size2index(size),
  33             false, tcache_get(tsd, true), true, NULL, true);
  34         if (quarantine == NULL)
  35                 return (NULL);
  36         quarantine->curbytes = 0;
  37         quarantine->curobjs = 0;
  38         quarantine->first = 0;
  39         quarantine->lg_maxobjs = lg_maxobjs;
  40
  41         return (quarantine);
  42 }
  43
  44 void
  45 quarantine_alloc_hook_work(tsd_t *tsd)
  46 {
  47         quarantine_t *quarantine;
  48
  49         if (!tsd_nominal(tsd))
  50                 return;
  51
  52         quarantine = quarantine_init(tsd, LG_MAXOBJS_INIT);
  53         /*
  54          * Check again whether quarantine has been initialized, because
  55          * quarantine_init() may have triggered recursive initialization.
  56          */
  57         if (tsd_quarantine_get(tsd) == NULL)
  58                 tsd_quarantine_set(tsd, quarantine);
  59         else
  60                 idalloctm(tsd, quarantine, tcache_get(tsd, false), true, true);
  61 }
  62
  63 static quarantine_t *
  64 quarantine_grow(tsd_t *tsd, quarantine_t *quarantine)
  65 {
  66         quarantine_t *ret;
  67
  68         ret = quarantine_init(tsd, quarantine->lg_maxobjs + 1);
  69         if (ret == NULL) {
  70                 quarantine_drain_one(tsd, quarantine);
  71                 return (quarantine);
  72         }
  73
  74         ret->curbytes = quarantine->curbytes;
  75         ret->curobjs = quarantine->curobjs;
  76         if (quarantine->first + quarantine->curobjs <= (ZU(1) <<
  77             quarantine->lg_maxobjs)) {
  78                 /* objs ring buffer data are contiguous. */
  79                 memcpy(ret->objs, &quarantine->objs[quarantine->first],
  80                     quarantine->curobjs * sizeof(quarantine_obj_t));
  81         } else {
  82                 /* objs ring buffer data wrap around. */
  83                 size_t ncopy_a = (ZU(1) << quarantine->lg_maxobjs) -
  84                     quarantine->first;
  85                 size_t ncopy_b = quarantine->curobjs - ncopy_a;
  86
  87                 memcpy(ret->objs, &quarantine->objs[quarantine->first], ncopy_a
  88                     * sizeof(quarantine_obj_t));
  89                 memcpy(&ret->objs[ncopy_a], quarantine->objs, ncopy_b *
  90                     sizeof(quarantine_obj_t));
  91         }
  92         idalloctm(tsd, quarantine, tcache_get(tsd, false), true, true);
  93
  94         tsd_quarantine_set(tsd, ret);
  95         return (ret);
  96 }
  97
  98 static void
  99 quarantine_drain_one(tsd_t *tsd, quarantine_t *quarantine)
 100 {
 101         quarantine_obj_t *obj = &quarantine->objs[quarantine->first];
 102         assert(obj->usize == isalloc(obj->ptr, config_prof));
 103         idalloctm(tsd, obj->ptr, NULL, false, true);
 104         quarantine->curbytes -= obj->usize;
 105         quarantine->curobjs--;
 106         quarantine->first = (quarantine->first + 1) & ((ZU(1) <<
 107             quarantine->lg_maxobjs) - 1);
 108 }
 109
 110 static void
 111 quarantine_drain(tsd_t *tsd, quarantine_t *quarantine, size_t upper_bound)
 112 {
 113
 114         while (quarantine->curbytes > upper_bound && quarantine->curobjs > 0)
 115                 quarantine_drain_one(tsd, quarantine);
 116 }
 117
 118 void
 119 quarantine(tsd_t *tsd, void *ptr)
 120 {
 121         quarantine_t *quarantine;
 122         size_t usize = isalloc(ptr, config_prof);
 123
 124         cassert(config_fill);
 125         assert(opt_quarantine);
 126
 127         if ((quarantine = tsd_quarantine_get(tsd)) == NULL) {
 128                 idalloctm(tsd, ptr, NULL, false, true);
 129                 return;
 130         }
 131         /*
 132          * Drain one or more objects if the quarantine size limit would be
 133          * exceeded by appending ptr.
 134          */
 135         if (quarantine->curbytes + usize > opt_quarantine) {
 136                 size_t upper_bound = (opt_quarantine >= usize) ? opt_quarantine
 137                     - usize : 0;
 138                 quarantine_drain(tsd, quarantine, upper_bound);
 139         }
 140         /* Grow the quarantine ring buffer if it's full. */
 141         if (quarantine->curobjs == (ZU(1) << quarantine->lg_maxobjs))
 142                 quarantine = quarantine_grow(tsd, quarantine);
 143         /* quarantine_grow() must free a slot if it fails to grow. */
 144         assert(quarantine->curobjs < (ZU(1) << quarantine->lg_maxobjs));
 145         /* Append ptr if its size doesn't exceed the quarantine size. */
 146         if (quarantine->curbytes + usize <= opt_quarantine) {
 147                 size_t offset = (quarantine->first + quarantine->curobjs) &
 148                     ((ZU(1) << quarantine->lg_maxobjs) - 1);
 149                 quarantine_obj_t *obj = &quarantine->objs[offset];
 150                 obj->ptr = ptr;
 151                 obj->usize = usize;
 152                 quarantine->curbytes += usize;
 153                 quarantine->curobjs++;
 154                 if (config_fill && unlikely(opt_junk_free)) {
 155                         /*
 156                          * Only do redzone validation if Valgrind isn't in
 157                          * operation.
 158                          */
 159                         if ((!config_valgrind || likely(!in_valgrind))
 160                             && usize <= SMALL_MAXCLASS)
 161                                 arena_quarantine_junk_small(ptr, usize);
 162                         else
 163                                 memset(ptr, 0x5a, usize);
 164                 }
 165         } else {
 166                 assert(quarantine->curbytes == 0);
 167                 idalloctm(tsd, ptr, NULL, false, true);
 168         }
 169 }
 170
 171 void
 172 quarantine_cleanup(tsd_t *tsd)
 173 {
 174         quarantine_t *quarantine;
 175
 176         if (!config_fill)
 177                 return;
 178
 179         quarantine = tsd_quarantine_get(tsd);
 180         if (quarantine != NULL) {
 181                 quarantine_drain(tsd, quarantine, 0);
 182                 idalloctm(tsd, quarantine, tcache_get(tsd, false), true, true);
 183                 tsd_quarantine_set(tsd, NULL);
 184         }
 185 }