]> git.proxmox.com Git - rustc.git/blame - src/jemalloc/src/base.c
Imported Upstream version 1.9.0+dfsg1
[rustc.git] / src / jemalloc / src / base.c
CommitLineData
970d7e83
LB
1#define JEMALLOC_BASE_C_
2#include "jemalloc/internal/jemalloc_internal.h"
3
4/******************************************************************************/
5/* Data. */
6
7static malloc_mutex_t base_mtx;
54a0048b 8static extent_tree_t base_avail_szad;
970d7e83 9static extent_node_t *base_nodes;
54a0048b
SL
10static size_t base_allocated;
11static size_t base_resident;
12static size_t base_mapped;
970d7e83 13
970d7e83
LB
14/******************************************************************************/
15
54a0048b
SL
16/* base_mtx must be held. */
17static extent_node_t *
18base_node_try_alloc(void)
970d7e83 19{
54a0048b 20 extent_node_t *node;
970d7e83 21
54a0048b
SL
22 if (base_nodes == NULL)
23 return (NULL);
24 node = base_nodes;
25 base_nodes = *(extent_node_t **)node;
26 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(node, sizeof(extent_node_t));
27 return (node);
970d7e83
LB
28}
29
54a0048b
SL
30/* base_mtx must be held. */
31static void
32base_node_dalloc(extent_node_t *node)
970d7e83 33{
970d7e83 34
54a0048b
SL
35 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(node, sizeof(extent_node_t));
36 *(extent_node_t **)node = base_nodes;
37 base_nodes = node;
970d7e83
LB
38}
39
54a0048b
SL
40/* base_mtx must be held. */
41static extent_node_t *
42base_chunk_alloc(size_t minsize)
970d7e83 43{
54a0048b
SL
44 extent_node_t *node;
45 size_t csize, nsize;
46 void *addr;
7453a54e 47
54a0048b
SL
48 assert(minsize != 0);
49 node = base_node_try_alloc();
50 /* Allocate enough space to also carve a node out if necessary. */
51 nsize = (node == NULL) ? CACHELINE_CEILING(sizeof(extent_node_t)) : 0;
52 csize = CHUNK_CEILING(minsize + nsize);
53 addr = chunk_alloc_base(csize);
54 if (addr == NULL) {
55 if (node != NULL)
56 base_node_dalloc(node);
57 return (NULL);
58 }
59 base_mapped += csize;
60 if (node == NULL) {
61 node = (extent_node_t *)addr;
62 addr = (void *)((uintptr_t)addr + nsize);
63 csize -= nsize;
64 if (config_stats) {
65 base_allocated += nsize;
66 base_resident += PAGE_CEILING(nsize);
67 }
68 }
69 extent_node_init(node, NULL, addr, csize, true, true);
70 return (node);
7453a54e
SL
71}
72
54a0048b
SL
73/*
74 * base_alloc() guarantees demand-zeroed memory, in order to make multi-page
75 * sparse data structures such as radix tree nodes efficient with respect to
76 * physical memory usage.
77 */
78void *
79base_alloc(size_t size)
7453a54e 80{
54a0048b
SL
81 void *ret;
82 size_t csize, usize;
83 extent_node_t *node;
84 extent_node_t key;
85
86 /*
87 * Round size up to nearest multiple of the cacheline size, so that
88 * there is no chance of false cache line sharing.
89 */
90 csize = CACHELINE_CEILING(size);
970d7e83 91
54a0048b
SL
92 usize = s2u(csize);
93 extent_node_init(&key, NULL, NULL, usize, false, false);
970d7e83 94 malloc_mutex_lock(&base_mtx);
54a0048b
SL
95 node = extent_tree_szad_nsearch(&base_avail_szad, &key);
96 if (node != NULL) {
97 /* Use existing space. */
98 extent_tree_szad_remove(&base_avail_szad, node);
970d7e83 99 } else {
54a0048b
SL
100 /* Try to allocate more space. */
101 node = base_chunk_alloc(csize);
102 }
103 if (node == NULL) {
104 ret = NULL;
105 goto label_return;
970d7e83
LB
106 }
107
54a0048b
SL
108 ret = extent_node_addr_get(node);
109 if (extent_node_size_get(node) > csize) {
110 extent_node_addr_set(node, (void *)((uintptr_t)ret + csize));
111 extent_node_size_set(node, extent_node_size_get(node) - csize);
112 extent_tree_szad_insert(&base_avail_szad, node);
113 } else
114 base_node_dalloc(node);
115 if (config_stats) {
116 base_allocated += csize;
117 /*
118 * Add one PAGE to base_resident for every page boundary that is
119 * crossed by the new allocation.
120 */
121 base_resident += PAGE_CEILING((uintptr_t)ret + csize) -
122 PAGE_CEILING((uintptr_t)ret);
123 }
124 JEMALLOC_VALGRIND_MAKE_MEM_DEFINED(ret, csize);
125label_return:
126 malloc_mutex_unlock(&base_mtx);
970d7e83
LB
127 return (ret);
128}
129
130void
54a0048b 131base_stats_get(size_t *allocated, size_t *resident, size_t *mapped)
970d7e83
LB
132{
133
970d7e83 134 malloc_mutex_lock(&base_mtx);
54a0048b
SL
135 assert(base_allocated <= base_resident);
136 assert(base_resident <= base_mapped);
137 *allocated = base_allocated;
138 *resident = base_resident;
139 *mapped = base_mapped;
970d7e83
LB
140 malloc_mutex_unlock(&base_mtx);
141}
142
143bool
144base_boot(void)
145{
146
970d7e83
LB
147 if (malloc_mutex_init(&base_mtx))
148 return (true);
54a0048b
SL
149 extent_tree_szad_new(&base_avail_szad);
150 base_nodes = NULL;
970d7e83
LB
151
152 return (false);
153}
154
155void
156base_prefork(void)
157{
158
159 malloc_mutex_prefork(&base_mtx);
160}
161
162void
163base_postfork_parent(void)
164{
165
166 malloc_mutex_postfork_parent(&base_mtx);
167}
168
169void
170base_postfork_child(void)
171{
172
173 malloc_mutex_postfork_child(&base_mtx);
174}