src/jemalloc/src/base.c

   1 #define JEMALLOC_BASE_C_
   2 #include "jemalloc/internal/jemalloc_internal.h"
   3
   4 /******************************************************************************/
   5 /* Data. */
   6
   7 static malloc_mutex_t   base_mtx;
   8 static size_t           base_extent_sn_next;
   9 static extent_tree_t    base_avail_szsnad;
  10 static extent_node_t    *base_nodes;
  11 static size_t           base_allocated;
  12 static size_t           base_resident;
  13 static size_t           base_mapped;
  14
  15 /******************************************************************************/
  16
  17 static extent_node_t *
  18 base_node_try_alloc(tsdn_t *tsdn)
  19 {
  20         extent_node_t *node;
  21
  22         malloc_mutex_assert_owner(tsdn, &base_mtx);
  23
  24         if (base_nodes == NULL)
  25                 return (NULL);
  26         node = base_nodes;
  27         base_nodes = *(extent_node_t **)node;
  28         JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(node, sizeof(extent_node_t));
  29         return (node);
  30 }
  31
  32 static void
  33 base_node_dalloc(tsdn_t *tsdn, extent_node_t *node)
  34 {
  35
  36         malloc_mutex_assert_owner(tsdn, &base_mtx);
  37
  38         JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(node, sizeof(extent_node_t));
  39         *(extent_node_t **)node = base_nodes;
  40         base_nodes = node;
  41 }
  42
  43 static void
  44 base_extent_node_init(extent_node_t *node, void *addr, size_t size)
  45 {
  46         size_t sn = atomic_add_z(&base_extent_sn_next, 1) - 1;
  47
  48         extent_node_init(node, NULL, addr, size, sn, true, true);
  49 }
  50
  51 static extent_node_t *
  52 base_chunk_alloc(tsdn_t *tsdn, size_t minsize)
  53 {
  54         extent_node_t *node;
  55         size_t csize, nsize;
  56         void *addr;
  57
  58         malloc_mutex_assert_owner(tsdn, &base_mtx);
  59         assert(minsize != 0);
  60         node = base_node_try_alloc(tsdn);
  61         /* Allocate enough space to also carve a node out if necessary. */
  62         nsize = (node == NULL) ? CACHELINE_CEILING(sizeof(extent_node_t)) : 0;
  63         csize = CHUNK_CEILING(minsize + nsize);
  64         addr = chunk_alloc_base(csize);
  65         if (addr == NULL) {
  66                 if (node != NULL)
  67                         base_node_dalloc(tsdn, node);
  68                 return (NULL);
  69         }
  70         base_mapped += csize;
  71         if (node == NULL) {
  72                 node = (extent_node_t *)addr;
  73                 addr = (void *)((uintptr_t)addr + nsize);
  74                 csize -= nsize;
  75                 if (config_stats) {
  76                         base_allocated += nsize;
  77                         base_resident += PAGE_CEILING(nsize);
  78                 }
  79         }
  80         base_extent_node_init(node, addr, csize);
  81         return (node);
  82 }
  83
  84 /*
  85  * base_alloc() guarantees demand-zeroed memory, in order to make multi-page
  86  * sparse data structures such as radix tree nodes efficient with respect to
  87  * physical memory usage.
  88  */
  89 void *
  90 base_alloc(tsdn_t *tsdn, size_t size)
  91 {
  92         void *ret;
  93         size_t csize, usize;
  94         extent_node_t *node;
  95         extent_node_t key;
  96
  97         /*
  98          * Round size up to nearest multiple of the cacheline size, so that
  99          * there is no chance of false cache line sharing.
 100          */
 101         csize = CACHELINE_CEILING(size);
 102
 103         usize = s2u(csize);
 104         extent_node_init(&key, NULL, NULL, usize, 0, false, false);
 105         malloc_mutex_lock(tsdn, &base_mtx);
 106         node = extent_tree_szsnad_nsearch(&base_avail_szsnad, &key);
 107         if (node != NULL) {
 108                 /* Use existing space. */
 109                 extent_tree_szsnad_remove(&base_avail_szsnad, node);
 110         } else {
 111                 /* Try to allocate more space. */
 112                 node = base_chunk_alloc(tsdn, csize);
 113         }
 114         if (node == NULL) {
 115                 ret = NULL;
 116                 goto label_return;
 117         }
 118
 119         ret = extent_node_addr_get(node);
 120         if (extent_node_size_get(node) > csize) {
 121                 extent_node_addr_set(node, (void *)((uintptr_t)ret + csize));
 122                 extent_node_size_set(node, extent_node_size_get(node) - csize);
 123                 extent_tree_szsnad_insert(&base_avail_szsnad, node);
 124         } else
 125                 base_node_dalloc(tsdn, node);
 126         if (config_stats) {
 127                 base_allocated += csize;
 128                 /*
 129                  * Add one PAGE to base_resident for every page boundary that is
 130                  * crossed by the new allocation.
 131                  */
 132                 base_resident += PAGE_CEILING((uintptr_t)ret + csize) -
 133                     PAGE_CEILING((uintptr_t)ret);
 134         }
 135         JEMALLOC_VALGRIND_MAKE_MEM_DEFINED(ret, csize);
 136 label_return:
 137         malloc_mutex_unlock(tsdn, &base_mtx);
 138         return (ret);
 139 }
 140
 141 void
 142 base_stats_get(tsdn_t *tsdn, size_t *allocated, size_t *resident,
 143     size_t *mapped)
 144 {
 145
 146         malloc_mutex_lock(tsdn, &base_mtx);
 147         assert(base_allocated <= base_resident);
 148         assert(base_resident <= base_mapped);
 149         *allocated = base_allocated;
 150         *resident = base_resident;
 151         *mapped = base_mapped;
 152         malloc_mutex_unlock(tsdn, &base_mtx);
 153 }
 154
 155 bool
 156 base_boot(void)
 157 {
 158
 159         if (malloc_mutex_init(&base_mtx, "base", WITNESS_RANK_BASE))
 160                 return (true);
 161         base_extent_sn_next = 0;
 162         extent_tree_szsnad_new(&base_avail_szsnad);
 163         base_nodes = NULL;
 164
 165         return (false);
 166 }
 167
 168 void
 169 base_prefork(tsdn_t *tsdn)
 170 {
 171
 172         malloc_mutex_prefork(tsdn, &base_mtx);
 173 }
 174
 175 void
 176 base_postfork_parent(tsdn_t *tsdn)
 177 {
 178
 179         malloc_mutex_postfork_parent(tsdn, &base_mtx);
 180 }
 181
 182 void
 183 base_postfork_child(tsdn_t *tsdn)
 184 {
 185
 186         malloc_mutex_postfork_child(tsdn, &base_mtx);
 187 }