]>
Commit | Line | Data |
---|---|---|
e9174d1e SL |
1 | // Copyright 2015 The Rust Project Developers. See the COPYRIGHT |
2 | // file at the top-level directory of this distribution and at | |
3 | // http://rust-lang.org/COPYRIGHT. | |
4 | // | |
5 | // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or | |
6 | // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license | |
7 | // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your | |
8 | // option. This file may not be copied, modified, or distributed | |
9 | // except according to those terms. | |
10 | ||
e9174d1e | 11 | #![no_std] |
ea8adc8c | 12 | #![allow(unused_attributes)] |
e9174d1e SL |
13 | #![unstable(feature = "alloc_jemalloc", |
14 | reason = "this library is unlikely to be stabilized in its current \ | |
15 | form or name", | |
16 | issue = "27783")] | |
32a655c1 | 17 | #![deny(warnings)] |
3b2f2976 XL |
18 | #![feature(alloc)] |
19 | #![feature(alloc_system)] | |
e9174d1e | 20 | #![feature(libc)] |
041b39d2 | 21 | #![feature(linkage)] |
3b2f2976 | 22 | #![feature(staged_api)] |
ea8adc8c | 23 | #![feature(rustc_attrs)] |
3b2f2976 XL |
24 | #![cfg_attr(dummy_jemalloc, allow(dead_code, unused_extern_crates))] |
25 | #![cfg_attr(not(dummy_jemalloc), feature(allocator_api))] | |
ea8adc8c | 26 | #![rustc_alloc_kind = "exe"] |
041b39d2 | 27 | |
041b39d2 | 28 | extern crate alloc; |
041b39d2 | 29 | extern crate alloc_system; |
e9174d1e SL |
30 | extern crate libc; |
31 | ||
3b2f2976 | 32 | #[cfg(not(dummy_jemalloc))] |
041b39d2 | 33 | pub use contents::*; |
3b2f2976 | 34 | #[cfg(not(dummy_jemalloc))] |
041b39d2 XL |
35 | mod contents { |
36 | use core::ptr; | |
e9174d1e | 37 | |
041b39d2 XL |
38 | use alloc::heap::{Alloc, AllocErr, Layout}; |
39 | use alloc_system::System; | |
c30ab7b3 | 40 | use libc::{c_int, c_void, size_t}; |
e9174d1e | 41 | |
cc61c64b | 42 | // Note that the symbols here are prefixed by default on macOS and Windows (we |
c30ab7b3 SL |
43 | // don't explicitly request it), and on Android and DragonFly we explicitly |
44 | // request it as unprefixing cause segfaults (mismatches in allocators). | |
45 | extern "C" { | |
46 | #[cfg_attr(any(target_os = "macos", target_os = "android", target_os = "ios", | |
cc61c64b | 47 | target_os = "dragonfly", target_os = "windows", target_env = "musl"), |
c30ab7b3 SL |
48 | link_name = "je_mallocx")] |
49 | fn mallocx(size: size_t, flags: c_int) -> *mut c_void; | |
50 | #[cfg_attr(any(target_os = "macos", target_os = "android", target_os = "ios", | |
cc61c64b XL |
51 | target_os = "dragonfly", target_os = "windows", target_env = "musl"), |
52 | link_name = "je_calloc")] | |
53 | fn calloc(size: size_t, flags: c_int) -> *mut c_void; | |
54 | #[cfg_attr(any(target_os = "macos", target_os = "android", target_os = "ios", | |
55 | target_os = "dragonfly", target_os = "windows", target_env = "musl"), | |
c30ab7b3 SL |
56 | link_name = "je_rallocx")] |
57 | fn rallocx(ptr: *mut c_void, size: size_t, flags: c_int) -> *mut c_void; | |
58 | #[cfg_attr(any(target_os = "macos", target_os = "android", target_os = "ios", | |
cc61c64b | 59 | target_os = "dragonfly", target_os = "windows", target_env = "musl"), |
c30ab7b3 SL |
60 | link_name = "je_xallocx")] |
61 | fn xallocx(ptr: *mut c_void, size: size_t, extra: size_t, flags: c_int) -> size_t; | |
62 | #[cfg_attr(any(target_os = "macos", target_os = "android", target_os = "ios", | |
cc61c64b | 63 | target_os = "dragonfly", target_os = "windows", target_env = "musl"), |
c30ab7b3 SL |
64 | link_name = "je_sdallocx")] |
65 | fn sdallocx(ptr: *mut c_void, size: size_t, flags: c_int); | |
66 | #[cfg_attr(any(target_os = "macos", target_os = "android", target_os = "ios", | |
cc61c64b | 67 | target_os = "dragonfly", target_os = "windows", target_env = "musl"), |
c30ab7b3 SL |
68 | link_name = "je_nallocx")] |
69 | fn nallocx(size: size_t, flags: c_int) -> size_t; | |
70 | } | |
71 | ||
cc61c64b XL |
72 | const MALLOCX_ZERO: c_int = 0x40; |
73 | ||
c30ab7b3 SL |
74 | // The minimum alignment guaranteed by the architecture. This value is used to |
75 | // add fast paths for low alignment values. In practice, the alignment is a | |
76 | // constant at the call site and the branch will be optimized out. | |
77 | #[cfg(all(any(target_arch = "arm", | |
78 | target_arch = "mips", | |
79 | target_arch = "powerpc")))] | |
80 | const MIN_ALIGN: usize = 8; | |
81 | #[cfg(all(any(target_arch = "x86", | |
82 | target_arch = "x86_64", | |
83 | target_arch = "aarch64", | |
84 | target_arch = "powerpc64", | |
85 | target_arch = "mips64", | |
32a655c1 SL |
86 | target_arch = "s390x", |
87 | target_arch = "sparc64")))] | |
c30ab7b3 SL |
88 | const MIN_ALIGN: usize = 16; |
89 | ||
90 | // MALLOCX_ALIGN(a) macro | |
91 | fn mallocx_align(a: usize) -> c_int { | |
92 | a.trailing_zeros() as c_int | |
93 | } | |
94 | ||
95 | fn align_to_flags(align: usize) -> c_int { | |
96 | if align <= MIN_ALIGN { | |
97 | 0 | |
98 | } else { | |
99 | mallocx_align(align) | |
100 | } | |
101 | } | |
102 | ||
041b39d2 XL |
103 | // for symbol names src/librustc/middle/allocator.rs |
104 | // for signatures src/librustc_allocator/lib.rs | |
c30ab7b3 | 105 | |
041b39d2 XL |
106 | // linkage directives are provided as part of the current compiler allocator |
107 | // ABI | |
cc61c64b | 108 | |
c30ab7b3 | 109 | #[no_mangle] |
abe05a73 | 110 | #[rustc_std_internal_symbol] |
041b39d2 XL |
111 | pub unsafe extern fn __rde_alloc(size: usize, |
112 | align: usize, | |
113 | err: *mut u8) -> *mut u8 { | |
c30ab7b3 | 114 | let flags = align_to_flags(align); |
041b39d2 XL |
115 | let ptr = mallocx(size as size_t, flags) as *mut u8; |
116 | if ptr.is_null() { | |
117 | let layout = Layout::from_size_align_unchecked(size, align); | |
118 | ptr::write(err as *mut AllocErr, | |
119 | AllocErr::Exhausted { request: layout }); | |
120 | } | |
121 | ptr | |
c30ab7b3 SL |
122 | } |
123 | ||
124 | #[no_mangle] | |
abe05a73 | 125 | #[rustc_std_internal_symbol] |
041b39d2 XL |
126 | pub unsafe extern fn __rde_oom(err: *const u8) -> ! { |
127 | System.oom((*(err as *const AllocErr)).clone()) | |
c30ab7b3 | 128 | } |
e9174d1e | 129 | |
c30ab7b3 | 130 | #[no_mangle] |
abe05a73 | 131 | #[rustc_std_internal_symbol] |
041b39d2 XL |
132 | pub unsafe extern fn __rde_dealloc(ptr: *mut u8, |
133 | size: usize, | |
134 | align: usize) { | |
c30ab7b3 | 135 | let flags = align_to_flags(align); |
041b39d2 | 136 | sdallocx(ptr as *mut c_void, size, flags); |
c30ab7b3 SL |
137 | } |
138 | ||
139 | #[no_mangle] | |
abe05a73 | 140 | #[rustc_std_internal_symbol] |
041b39d2 XL |
141 | pub unsafe extern fn __rde_usable_size(layout: *const u8, |
142 | min: *mut usize, | |
143 | max: *mut usize) { | |
144 | let layout = &*(layout as *const Layout); | |
145 | let flags = align_to_flags(layout.align()); | |
146 | let size = nallocx(layout.size(), flags) as usize; | |
147 | *min = layout.size(); | |
148 | if size > 0 { | |
149 | *max = size; | |
150 | } else { | |
151 | *max = layout.size(); | |
152 | } | |
c30ab7b3 | 153 | } |
e9174d1e | 154 | |
c30ab7b3 | 155 | #[no_mangle] |
abe05a73 | 156 | #[rustc_std_internal_symbol] |
041b39d2 XL |
157 | pub unsafe extern fn __rde_realloc(ptr: *mut u8, |
158 | _old_size: usize, | |
159 | old_align: usize, | |
160 | new_size: usize, | |
161 | new_align: usize, | |
162 | err: *mut u8) -> *mut u8 { | |
163 | if new_align != old_align { | |
164 | ptr::write(err as *mut AllocErr, | |
165 | AllocErr::Unsupported { details: "can't change alignments" }); | |
166 | return 0 as *mut u8 | |
167 | } | |
168 | ||
169 | let flags = align_to_flags(new_align); | |
170 | let ptr = rallocx(ptr as *mut c_void, new_size, flags) as *mut u8; | |
171 | if ptr.is_null() { | |
172 | let layout = Layout::from_size_align_unchecked(new_size, new_align); | |
173 | ptr::write(err as *mut AllocErr, | |
174 | AllocErr::Exhausted { request: layout }); | |
175 | } | |
176 | ptr | |
c30ab7b3 | 177 | } |
e9174d1e | 178 | |
cc61c64b | 179 | #[no_mangle] |
abe05a73 | 180 | #[rustc_std_internal_symbol] |
041b39d2 XL |
181 | pub unsafe extern fn __rde_alloc_zeroed(size: usize, |
182 | align: usize, | |
183 | err: *mut u8) -> *mut u8 { | |
184 | let ptr = if align <= MIN_ALIGN { | |
185 | calloc(size as size_t, 1) as *mut u8 | |
186 | } else { | |
187 | let flags = align_to_flags(align) | MALLOCX_ZERO; | |
188 | mallocx(size as size_t, flags) as *mut u8 | |
189 | }; | |
190 | if ptr.is_null() { | |
191 | let layout = Layout::from_size_align_unchecked(size, align); | |
192 | ptr::write(err as *mut AllocErr, | |
193 | AllocErr::Exhausted { request: layout }); | |
194 | } | |
195 | ptr | |
cc61c64b XL |
196 | } |
197 | ||
c30ab7b3 | 198 | #[no_mangle] |
abe05a73 | 199 | #[rustc_std_internal_symbol] |
041b39d2 XL |
200 | pub unsafe extern fn __rde_alloc_excess(size: usize, |
201 | align: usize, | |
202 | excess: *mut usize, | |
203 | err: *mut u8) -> *mut u8 { | |
204 | let p = __rde_alloc(size, align, err); | |
205 | if !p.is_null() { | |
abe05a73 XL |
206 | let flags = align_to_flags(align); |
207 | *excess = nallocx(size, flags) as usize; | |
041b39d2 XL |
208 | } |
209 | return p | |
c30ab7b3 | 210 | } |
e9174d1e | 211 | |
c30ab7b3 | 212 | #[no_mangle] |
abe05a73 | 213 | #[rustc_std_internal_symbol] |
041b39d2 XL |
214 | pub unsafe extern fn __rde_realloc_excess(ptr: *mut u8, |
215 | old_size: usize, | |
216 | old_align: usize, | |
217 | new_size: usize, | |
218 | new_align: usize, | |
219 | excess: *mut usize, | |
220 | err: *mut u8) -> *mut u8 { | |
221 | let p = __rde_realloc(ptr, old_size, old_align, new_size, new_align, err); | |
222 | if !p.is_null() { | |
abe05a73 XL |
223 | let flags = align_to_flags(new_align); |
224 | *excess = nallocx(new_size, flags) as usize; | |
041b39d2 | 225 | } |
abe05a73 | 226 | p |
c30ab7b3 | 227 | } |
e9174d1e | 228 | |
c30ab7b3 | 229 | #[no_mangle] |
abe05a73 | 230 | #[rustc_std_internal_symbol] |
041b39d2 XL |
231 | pub unsafe extern fn __rde_grow_in_place(ptr: *mut u8, |
232 | old_size: usize, | |
233 | old_align: usize, | |
234 | new_size: usize, | |
235 | new_align: usize) -> u8 { | |
236 | __rde_shrink_in_place(ptr, old_size, old_align, new_size, new_align) | |
c30ab7b3 | 237 | } |
9cc50fc6 | 238 | |
c30ab7b3 | 239 | #[no_mangle] |
abe05a73 | 240 | #[rustc_std_internal_symbol] |
041b39d2 XL |
241 | pub unsafe extern fn __rde_shrink_in_place(ptr: *mut u8, |
242 | _old_size: usize, | |
243 | old_align: usize, | |
244 | new_size: usize, | |
245 | new_align: usize) -> u8 { | |
246 | if old_align == new_align { | |
247 | let flags = align_to_flags(new_align); | |
248 | (xallocx(ptr as *mut c_void, new_size, 0, flags) == new_size) as u8 | |
249 | } else { | |
250 | 0 | |
251 | } | |
c30ab7b3 | 252 | } |
9cc50fc6 | 253 | } |