]>
Commit | Line | Data |
---|---|---|
970d7e83 LB |
1 | #define JEMALLOC_BASE_C_ |
2 | #include "jemalloc/internal/jemalloc_internal.h" | |
3 | ||
4 | /******************************************************************************/ | |
5 | /* Data. */ | |
6 | ||
7 | static malloc_mutex_t base_mtx; | |
54a0048b | 8 | static extent_tree_t base_avail_szad; |
970d7e83 | 9 | static extent_node_t *base_nodes; |
54a0048b SL |
10 | static size_t base_allocated; |
11 | static size_t base_resident; | |
12 | static size_t base_mapped; | |
970d7e83 | 13 | |
970d7e83 LB |
14 | /******************************************************************************/ |
15 | ||
54a0048b SL |
16 | /* base_mtx must be held. */ |
17 | static extent_node_t * | |
18 | base_node_try_alloc(void) | |
970d7e83 | 19 | { |
54a0048b | 20 | extent_node_t *node; |
970d7e83 | 21 | |
54a0048b SL |
22 | if (base_nodes == NULL) |
23 | return (NULL); | |
24 | node = base_nodes; | |
25 | base_nodes = *(extent_node_t **)node; | |
26 | JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(node, sizeof(extent_node_t)); | |
27 | return (node); | |
970d7e83 LB |
28 | } |
29 | ||
54a0048b SL |
30 | /* base_mtx must be held. */ |
31 | static void | |
32 | base_node_dalloc(extent_node_t *node) | |
970d7e83 | 33 | { |
970d7e83 | 34 | |
54a0048b SL |
35 | JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(node, sizeof(extent_node_t)); |
36 | *(extent_node_t **)node = base_nodes; | |
37 | base_nodes = node; | |
970d7e83 LB |
38 | } |
39 | ||
54a0048b SL |
40 | /* base_mtx must be held. */ |
41 | static extent_node_t * | |
42 | base_chunk_alloc(size_t minsize) | |
970d7e83 | 43 | { |
54a0048b SL |
44 | extent_node_t *node; |
45 | size_t csize, nsize; | |
46 | void *addr; | |
7453a54e | 47 | |
54a0048b SL |
48 | assert(minsize != 0); |
49 | node = base_node_try_alloc(); | |
50 | /* Allocate enough space to also carve a node out if necessary. */ | |
51 | nsize = (node == NULL) ? CACHELINE_CEILING(sizeof(extent_node_t)) : 0; | |
52 | csize = CHUNK_CEILING(minsize + nsize); | |
53 | addr = chunk_alloc_base(csize); | |
54 | if (addr == NULL) { | |
55 | if (node != NULL) | |
56 | base_node_dalloc(node); | |
57 | return (NULL); | |
58 | } | |
59 | base_mapped += csize; | |
60 | if (node == NULL) { | |
61 | node = (extent_node_t *)addr; | |
62 | addr = (void *)((uintptr_t)addr + nsize); | |
63 | csize -= nsize; | |
64 | if (config_stats) { | |
65 | base_allocated += nsize; | |
66 | base_resident += PAGE_CEILING(nsize); | |
67 | } | |
68 | } | |
69 | extent_node_init(node, NULL, addr, csize, true, true); | |
70 | return (node); | |
7453a54e SL |
71 | } |
72 | ||
54a0048b SL |
73 | /* |
74 | * base_alloc() guarantees demand-zeroed memory, in order to make multi-page | |
75 | * sparse data structures such as radix tree nodes efficient with respect to | |
76 | * physical memory usage. | |
77 | */ | |
78 | void * | |
79 | base_alloc(size_t size) | |
7453a54e | 80 | { |
54a0048b SL |
81 | void *ret; |
82 | size_t csize, usize; | |
83 | extent_node_t *node; | |
84 | extent_node_t key; | |
85 | ||
86 | /* | |
87 | * Round size up to nearest multiple of the cacheline size, so that | |
88 | * there is no chance of false cache line sharing. | |
89 | */ | |
90 | csize = CACHELINE_CEILING(size); | |
970d7e83 | 91 | |
54a0048b SL |
92 | usize = s2u(csize); |
93 | extent_node_init(&key, NULL, NULL, usize, false, false); | |
970d7e83 | 94 | malloc_mutex_lock(&base_mtx); |
54a0048b SL |
95 | node = extent_tree_szad_nsearch(&base_avail_szad, &key); |
96 | if (node != NULL) { | |
97 | /* Use existing space. */ | |
98 | extent_tree_szad_remove(&base_avail_szad, node); | |
970d7e83 | 99 | } else { |
54a0048b SL |
100 | /* Try to allocate more space. */ |
101 | node = base_chunk_alloc(csize); | |
102 | } | |
103 | if (node == NULL) { | |
104 | ret = NULL; | |
105 | goto label_return; | |
970d7e83 LB |
106 | } |
107 | ||
54a0048b SL |
108 | ret = extent_node_addr_get(node); |
109 | if (extent_node_size_get(node) > csize) { | |
110 | extent_node_addr_set(node, (void *)((uintptr_t)ret + csize)); | |
111 | extent_node_size_set(node, extent_node_size_get(node) - csize); | |
112 | extent_tree_szad_insert(&base_avail_szad, node); | |
113 | } else | |
114 | base_node_dalloc(node); | |
115 | if (config_stats) { | |
116 | base_allocated += csize; | |
117 | /* | |
118 | * Add one PAGE to base_resident for every page boundary that is | |
119 | * crossed by the new allocation. | |
120 | */ | |
121 | base_resident += PAGE_CEILING((uintptr_t)ret + csize) - | |
122 | PAGE_CEILING((uintptr_t)ret); | |
123 | } | |
124 | JEMALLOC_VALGRIND_MAKE_MEM_DEFINED(ret, csize); | |
125 | label_return: | |
126 | malloc_mutex_unlock(&base_mtx); | |
970d7e83 LB |
127 | return (ret); |
128 | } | |
129 | ||
130 | void | |
54a0048b | 131 | base_stats_get(size_t *allocated, size_t *resident, size_t *mapped) |
970d7e83 LB |
132 | { |
133 | ||
970d7e83 | 134 | malloc_mutex_lock(&base_mtx); |
54a0048b SL |
135 | assert(base_allocated <= base_resident); |
136 | assert(base_resident <= base_mapped); | |
137 | *allocated = base_allocated; | |
138 | *resident = base_resident; | |
139 | *mapped = base_mapped; | |
970d7e83 LB |
140 | malloc_mutex_unlock(&base_mtx); |
141 | } | |
142 | ||
143 | bool | |
144 | base_boot(void) | |
145 | { | |
146 | ||
970d7e83 LB |
147 | if (malloc_mutex_init(&base_mtx)) |
148 | return (true); | |
54a0048b SL |
149 | extent_tree_szad_new(&base_avail_szad); |
150 | base_nodes = NULL; | |
970d7e83 LB |
151 | |
152 | return (false); | |
153 | } | |
154 | ||
155 | void | |
156 | base_prefork(void) | |
157 | { | |
158 | ||
159 | malloc_mutex_prefork(&base_mtx); | |
160 | } | |
161 | ||
162 | void | |
163 | base_postfork_parent(void) | |
164 | { | |
165 | ||
166 | malloc_mutex_postfork_parent(&base_mtx); | |
167 | } | |
168 | ||
169 | void | |
170 | base_postfork_child(void) | |
171 | { | |
172 | ||
173 | malloc_mutex_postfork_child(&base_mtx); | |
174 | } |