]>
Commit | Line | Data |
---|---|---|
970d7e83 LB |
1 | #define JEMALLOC_CHUNK_DSS_C_ |
2 | #include "jemalloc/internal/jemalloc_internal.h" | |
3 | /******************************************************************************/ | |
4 | /* Data. */ | |
5 | ||
6 | const char *dss_prec_names[] = { | |
7 | "disabled", | |
8 | "primary", | |
9 | "secondary", | |
10 | "N/A" | |
11 | }; | |
12 | ||
13 | /* Current dss precedence default, used when creating new arenas. */ | |
14 | static dss_prec_t dss_prec_default = DSS_PREC_DEFAULT; | |
15 | ||
16 | /* | |
17 | * Protects sbrk() calls. This avoids malloc races among threads, though it | |
18 | * does not protect against races with threads that call sbrk() directly. | |
19 | */ | |
20 | static malloc_mutex_t dss_mtx; | |
21 | ||
22 | /* Base address of the DSS. */ | |
23 | static void *dss_base; | |
24 | /* Current end of the DSS, or ((void *)-1) if the DSS is exhausted. */ | |
25 | static void *dss_prev; | |
26 | /* Current upper limit on DSS addresses. */ | |
27 | static void *dss_max; | |
28 | ||
29 | /******************************************************************************/ | |
30 | ||
970d7e83 | 31 | static void * |
1a4d82fc | 32 | chunk_dss_sbrk(intptr_t increment) |
970d7e83 LB |
33 | { |
34 | ||
1a4d82fc JJ |
35 | #ifdef JEMALLOC_DSS |
36 | return (sbrk(increment)); | |
37 | #else | |
970d7e83 | 38 | not_implemented(); |
970d7e83 | 39 | return (NULL); |
970d7e83 | 40 | #endif |
1a4d82fc | 41 | } |
970d7e83 LB |
42 | |
43 | dss_prec_t | |
44 | chunk_dss_prec_get(void) | |
45 | { | |
46 | dss_prec_t ret; | |
47 | ||
1a4d82fc | 48 | if (!have_dss) |
970d7e83 LB |
49 | return (dss_prec_disabled); |
50 | malloc_mutex_lock(&dss_mtx); | |
51 | ret = dss_prec_default; | |
52 | malloc_mutex_unlock(&dss_mtx); | |
53 | return (ret); | |
54 | } | |
55 | ||
56 | bool | |
57 | chunk_dss_prec_set(dss_prec_t dss_prec) | |
58 | { | |
59 | ||
1a4d82fc JJ |
60 | if (!have_dss) |
61 | return (dss_prec != dss_prec_disabled); | |
970d7e83 LB |
62 | malloc_mutex_lock(&dss_mtx); |
63 | dss_prec_default = dss_prec; | |
64 | malloc_mutex_unlock(&dss_mtx); | |
65 | return (false); | |
66 | } | |
67 | ||
68 | void * | |
54a0048b SL |
69 | chunk_alloc_dss(arena_t *arena, void *new_addr, size_t size, size_t alignment, |
70 | bool *zero, bool *commit) | |
970d7e83 | 71 | { |
1a4d82fc | 72 | cassert(have_dss); |
970d7e83 LB |
73 | assert(size > 0 && (size & chunksize_mask) == 0); |
74 | assert(alignment > 0 && (alignment & chunksize_mask) == 0); | |
75 | ||
76 | /* | |
77 | * sbrk() uses a signed increment argument, so take care not to | |
78 | * interpret a huge allocation request as a negative increment. | |
79 | */ | |
80 | if ((intptr_t)size < 0) | |
81 | return (NULL); | |
82 | ||
83 | malloc_mutex_lock(&dss_mtx); | |
84 | if (dss_prev != (void *)-1) { | |
970d7e83 LB |
85 | |
86 | /* | |
87 | * The loop is necessary to recover from races with other | |
88 | * threads that are using the DSS for something other than | |
89 | * malloc. | |
90 | */ | |
91 | do { | |
54a0048b SL |
92 | void *ret, *cpad, *dss_next; |
93 | size_t gap_size, cpad_size; | |
94 | intptr_t incr; | |
95 | /* Avoid an unnecessary system call. */ | |
96 | if (new_addr != NULL && dss_max != new_addr) | |
97 | break; | |
98 | ||
970d7e83 | 99 | /* Get the current end of the DSS. */ |
1a4d82fc | 100 | dss_max = chunk_dss_sbrk(0); |
54a0048b SL |
101 | |
102 | /* Make sure the earlier condition still holds. */ | |
103 | if (new_addr != NULL && dss_max != new_addr) | |
104 | break; | |
105 | ||
970d7e83 LB |
106 | /* |
107 | * Calculate how much padding is necessary to | |
108 | * chunk-align the end of the DSS. | |
109 | */ | |
110 | gap_size = (chunksize - CHUNK_ADDR2OFFSET(dss_max)) & | |
111 | chunksize_mask; | |
112 | /* | |
113 | * Compute how much chunk-aligned pad space (if any) is | |
114 | * necessary to satisfy alignment. This space can be | |
115 | * recycled for later use. | |
116 | */ | |
117 | cpad = (void *)((uintptr_t)dss_max + gap_size); | |
118 | ret = (void *)ALIGNMENT_CEILING((uintptr_t)dss_max, | |
119 | alignment); | |
120 | cpad_size = (uintptr_t)ret - (uintptr_t)cpad; | |
121 | dss_next = (void *)((uintptr_t)ret + size); | |
122 | if ((uintptr_t)ret < (uintptr_t)dss_max || | |
123 | (uintptr_t)dss_next < (uintptr_t)dss_max) { | |
124 | /* Wrap-around. */ | |
125 | malloc_mutex_unlock(&dss_mtx); | |
126 | return (NULL); | |
127 | } | |
128 | incr = gap_size + cpad_size + size; | |
1a4d82fc | 129 | dss_prev = chunk_dss_sbrk(incr); |
970d7e83 LB |
130 | if (dss_prev == dss_max) { |
131 | /* Success. */ | |
132 | dss_max = dss_next; | |
133 | malloc_mutex_unlock(&dss_mtx); | |
54a0048b SL |
134 | if (cpad_size != 0) { |
135 | chunk_hooks_t chunk_hooks = | |
136 | CHUNK_HOOKS_INITIALIZER; | |
137 | chunk_dalloc_wrapper(arena, | |
138 | &chunk_hooks, cpad, cpad_size, | |
139 | true); | |
140 | } | |
970d7e83 | 141 | if (*zero) { |
1a4d82fc JJ |
142 | JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED( |
143 | ret, size); | |
970d7e83 LB |
144 | memset(ret, 0, size); |
145 | } | |
54a0048b SL |
146 | if (!*commit) |
147 | *commit = pages_decommit(ret, size); | |
970d7e83 LB |
148 | return (ret); |
149 | } | |
150 | } while (dss_prev != (void *)-1); | |
151 | } | |
152 | malloc_mutex_unlock(&dss_mtx); | |
153 | ||
154 | return (NULL); | |
155 | } | |
156 | ||
157 | bool | |
158 | chunk_in_dss(void *chunk) | |
159 | { | |
160 | bool ret; | |
161 | ||
1a4d82fc | 162 | cassert(have_dss); |
970d7e83 LB |
163 | |
164 | malloc_mutex_lock(&dss_mtx); | |
165 | if ((uintptr_t)chunk >= (uintptr_t)dss_base | |
166 | && (uintptr_t)chunk < (uintptr_t)dss_max) | |
167 | ret = true; | |
168 | else | |
169 | ret = false; | |
170 | malloc_mutex_unlock(&dss_mtx); | |
171 | ||
172 | return (ret); | |
173 | } | |
174 | ||
175 | bool | |
176 | chunk_dss_boot(void) | |
177 | { | |
178 | ||
1a4d82fc | 179 | cassert(have_dss); |
970d7e83 LB |
180 | |
181 | if (malloc_mutex_init(&dss_mtx)) | |
182 | return (true); | |
1a4d82fc | 183 | dss_base = chunk_dss_sbrk(0); |
970d7e83 LB |
184 | dss_prev = dss_base; |
185 | dss_max = dss_base; | |
186 | ||
187 | return (false); | |
188 | } | |
189 | ||
190 | void | |
191 | chunk_dss_prefork(void) | |
192 | { | |
193 | ||
1a4d82fc | 194 | if (have_dss) |
970d7e83 LB |
195 | malloc_mutex_prefork(&dss_mtx); |
196 | } | |
197 | ||
198 | void | |
199 | chunk_dss_postfork_parent(void) | |
200 | { | |
201 | ||
1a4d82fc | 202 | if (have_dss) |
970d7e83 LB |
203 | malloc_mutex_postfork_parent(&dss_mtx); |
204 | } | |
205 | ||
206 | void | |
207 | chunk_dss_postfork_child(void) | |
208 | { | |
209 | ||
1a4d82fc | 210 | if (have_dss) |
970d7e83 LB |
211 | malloc_mutex_postfork_child(&dss_mtx); |
212 | } | |
213 | ||
214 | /******************************************************************************/ |