]>
Commit | Line | Data |
---|---|---|
e9174d1e SL |
1 | // Copyright 2015 The Rust Project Developers. See the COPYRIGHT |
2 | // file at the top-level directory of this distribution and at | |
3 | // http://rust-lang.org/COPYRIGHT. | |
4 | // | |
5 | // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or | |
6 | // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license | |
7 | // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your | |
8 | // option. This file may not be copied, modified, or distributed | |
9 | // except according to those terms. | |
10 | ||
e9174d1e | 11 | #![no_std] |
ea8adc8c | 12 | #![allow(unused_attributes)] |
e9174d1e SL |
13 | #![unstable(feature = "alloc_system", |
14 | reason = "this library is unlikely to be stabilized in its current \ | |
15 | form or name", | |
abe05a73 | 16 | issue = "32838")] |
3b2f2976 XL |
17 | #![feature(global_allocator)] |
18 | #![feature(allocator_api)] | |
3b2f2976 | 19 | #![feature(core_intrinsics)] |
e9174d1e | 20 | #![feature(staged_api)] |
ea8adc8c | 21 | #![feature(rustc_attrs)] |
2c00a5a8 | 22 | #![cfg_attr(any(unix, target_os = "cloudabi", target_os = "redox"), feature(libc))] |
ea8adc8c | 23 | #![rustc_alloc_kind = "lib"] |
e9174d1e SL |
24 | |
25 | // The minimum alignment guaranteed by the architecture. This value is used to | |
ff7c6d11 | 26 | // add fast paths for low alignment values. |
92a42be0 SL |
27 | #[cfg(all(any(target_arch = "x86", |
28 | target_arch = "arm", | |
e9174d1e | 29 | target_arch = "mips", |
9cc50fc6 SL |
30 | target_arch = "powerpc", |
31 | target_arch = "powerpc64", | |
c30ab7b3 SL |
32 | target_arch = "asmjs", |
33 | target_arch = "wasm32")))] | |
abe05a73 | 34 | #[allow(dead_code)] |
e9174d1e | 35 | const MIN_ALIGN: usize = 8; |
92a42be0 | 36 | #[cfg(all(any(target_arch = "x86_64", |
9e0c209e SL |
37 | target_arch = "aarch64", |
38 | target_arch = "mips64", | |
32a655c1 SL |
39 | target_arch = "s390x", |
40 | target_arch = "sparc64")))] | |
abe05a73 | 41 | #[allow(dead_code)] |
e9174d1e SL |
42 | const MIN_ALIGN: usize = 16; |
43 | ||
83c7162d XL |
44 | use core::alloc::{Alloc, GlobalAlloc, AllocErr, Layout, Opaque}; |
45 | use core::ptr::NonNull; | |
cc61c64b | 46 | |
3b2f2976 XL |
47 | #[unstable(feature = "allocator_api", issue = "32838")] |
48 | pub struct System; | |
e9174d1e | 49 | |
3b2f2976 XL |
50 | #[unstable(feature = "allocator_api", issue = "32838")] |
51 | unsafe impl Alloc for System { | |
52 | #[inline] | |
83c7162d XL |
53 | unsafe fn alloc(&mut self, layout: Layout) -> Result<NonNull<Opaque>, AllocErr> { |
54 | NonNull::new(GlobalAlloc::alloc(self, layout)).ok_or(AllocErr) | |
3b2f2976 | 55 | } |
041b39d2 | 56 | |
3b2f2976 | 57 | #[inline] |
83c7162d XL |
58 | unsafe fn alloc_zeroed(&mut self, layout: Layout) -> Result<NonNull<Opaque>, AllocErr> { |
59 | NonNull::new(GlobalAlloc::alloc_zeroed(self, layout)).ok_or(AllocErr) | |
3b2f2976 | 60 | } |
041b39d2 | 61 | |
3b2f2976 | 62 | #[inline] |
83c7162d XL |
63 | unsafe fn dealloc(&mut self, ptr: NonNull<Opaque>, layout: Layout) { |
64 | GlobalAlloc::dealloc(self, ptr.as_ptr(), layout) | |
3b2f2976 | 65 | } |
041b39d2 | 66 | |
3b2f2976 XL |
67 | #[inline] |
68 | unsafe fn realloc(&mut self, | |
83c7162d XL |
69 | ptr: NonNull<Opaque>, |
70 | layout: Layout, | |
71 | new_size: usize) -> Result<NonNull<Opaque>, AllocErr> { | |
72 | NonNull::new(GlobalAlloc::realloc(self, ptr.as_ptr(), layout, new_size)).ok_or(AllocErr) | |
3b2f2976 | 73 | } |
83c7162d | 74 | } |
041b39d2 | 75 | |
83c7162d XL |
76 | #[cfg(stage0)] |
77 | #[unstable(feature = "allocator_api", issue = "32838")] | |
78 | unsafe impl<'a> Alloc for &'a System { | |
3b2f2976 | 79 | #[inline] |
83c7162d XL |
80 | unsafe fn alloc(&mut self, layout: Layout) -> Result<NonNull<Opaque>, AllocErr> { |
81 | NonNull::new(GlobalAlloc::alloc(*self, layout)).ok_or(AllocErr) | |
3b2f2976 | 82 | } |
041b39d2 | 83 | |
3b2f2976 | 84 | #[inline] |
83c7162d XL |
85 | unsafe fn alloc_zeroed(&mut self, layout: Layout) -> Result<NonNull<Opaque>, AllocErr> { |
86 | NonNull::new(GlobalAlloc::alloc_zeroed(*self, layout)).ok_or(AllocErr) | |
3b2f2976 | 87 | } |
041b39d2 | 88 | |
3b2f2976 | 89 | #[inline] |
83c7162d XL |
90 | unsafe fn dealloc(&mut self, ptr: NonNull<Opaque>, layout: Layout) { |
91 | GlobalAlloc::dealloc(*self, ptr.as_ptr(), layout) | |
3b2f2976 | 92 | } |
041b39d2 | 93 | |
3b2f2976 | 94 | #[inline] |
83c7162d XL |
95 | unsafe fn realloc(&mut self, |
96 | ptr: NonNull<Opaque>, | |
97 | layout: Layout, | |
98 | new_size: usize) -> Result<NonNull<Opaque>, AllocErr> { | |
99 | NonNull::new(GlobalAlloc::realloc(*self, ptr.as_ptr(), layout, new_size)).ok_or(AllocErr) | |
3b2f2976 | 100 | } |
83c7162d | 101 | } |
041b39d2 | 102 | |
83c7162d XL |
103 | #[cfg(any(windows, unix, target_os = "cloudabi", target_os = "redox"))] |
104 | mod realloc_fallback { | |
105 | use core::alloc::{GlobalAlloc, Opaque, Layout}; | |
106 | use core::cmp; | |
107 | use core::ptr; | |
108 | ||
109 | impl super::System { | |
110 | pub(crate) unsafe fn realloc_fallback(&self, ptr: *mut Opaque, old_layout: Layout, | |
111 | new_size: usize) -> *mut Opaque { | |
112 | // Docs for GlobalAlloc::realloc require this to be valid: | |
113 | let new_layout = Layout::from_size_align_unchecked(new_size, old_layout.align()); | |
114 | ||
115 | let new_ptr = GlobalAlloc::alloc(self, new_layout); | |
116 | if !new_ptr.is_null() { | |
117 | let size = cmp::min(old_layout.size(), new_size); | |
118 | ptr::copy_nonoverlapping(ptr as *mut u8, new_ptr as *mut u8, size); | |
119 | GlobalAlloc::dealloc(self, ptr, old_layout); | |
120 | } | |
121 | new_ptr | |
122 | } | |
041b39d2 | 123 | } |
e9174d1e SL |
124 | } |
125 | ||
2c00a5a8 | 126 | #[cfg(any(unix, target_os = "cloudabi", target_os = "redox"))] |
041b39d2 | 127 | mod platform { |
a7813a04 XL |
128 | extern crate libc; |
129 | ||
e9174d1e | 130 | use core::ptr; |
041b39d2 | 131 | |
e9174d1e | 132 | use MIN_ALIGN; |
3b2f2976 | 133 | use System; |
83c7162d | 134 | use core::alloc::{GlobalAlloc, Layout, Opaque}; |
041b39d2 XL |
135 | |
136 | #[unstable(feature = "allocator_api", issue = "32838")] | |
83c7162d | 137 | unsafe impl GlobalAlloc for System { |
041b39d2 | 138 | #[inline] |
83c7162d XL |
139 | unsafe fn alloc(&self, layout: Layout) -> *mut Opaque { |
140 | if layout.align() <= MIN_ALIGN && layout.align() <= layout.size() { | |
141 | libc::malloc(layout.size()) as *mut Opaque | |
041b39d2 | 142 | } else { |
83c7162d XL |
143 | #[cfg(target_os = "macos")] |
144 | { | |
145 | if layout.align() > (1 << 31) { | |
146 | // FIXME: use Opaque::null_mut | |
147 | // https://github.com/rust-lang/rust/issues/49659 | |
148 | return 0 as *mut Opaque | |
149 | } | |
150 | } | |
041b39d2 | 151 | aligned_malloc(&layout) |
041b39d2 XL |
152 | } |
153 | } | |
e9174d1e | 154 | |
041b39d2 | 155 | #[inline] |
83c7162d | 156 | unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut Opaque { |
ff7c6d11 | 157 | if layout.align() <= MIN_ALIGN && layout.align() <= layout.size() { |
83c7162d | 158 | libc::calloc(layout.size(), 1) as *mut Opaque |
041b39d2 | 159 | } else { |
83c7162d XL |
160 | let ptr = self.alloc(layout.clone()); |
161 | if !ptr.is_null() { | |
162 | ptr::write_bytes(ptr as *mut u8, 0, layout.size()); | |
041b39d2 | 163 | } |
83c7162d | 164 | ptr |
041b39d2 XL |
165 | } |
166 | } | |
167 | ||
168 | #[inline] | |
83c7162d | 169 | unsafe fn dealloc(&self, ptr: *mut Opaque, _layout: Layout) { |
041b39d2 XL |
170 | libc::free(ptr as *mut libc::c_void) |
171 | } | |
172 | ||
173 | #[inline] | |
83c7162d XL |
174 | unsafe fn realloc(&self, ptr: *mut Opaque, layout: Layout, new_size: usize) -> *mut Opaque { |
175 | if layout.align() <= MIN_ALIGN && layout.align() <= new_size { | |
176 | libc::realloc(ptr as *mut libc::c_void, new_size) as *mut Opaque | |
041b39d2 | 177 | } else { |
83c7162d | 178 | self.realloc_fallback(ptr, layout, new_size) |
041b39d2 | 179 | } |
a7813a04 XL |
180 | } |
181 | } | |
182 | ||
ea8adc8c | 183 | #[cfg(any(target_os = "android", target_os = "redox", target_os = "solaris"))] |
041b39d2 | 184 | #[inline] |
83c7162d | 185 | unsafe fn aligned_malloc(layout: &Layout) -> *mut Opaque { |
a7813a04 XL |
186 | // On android we currently target API level 9 which unfortunately |
187 | // doesn't have the `posix_memalign` API used below. Instead we use | |
188 | // `memalign`, but this unfortunately has the property on some systems | |
189 | // where the memory returned cannot be deallocated by `free`! | |
190 | // | |
191 | // Upon closer inspection, however, this appears to work just fine with | |
192 | // Android, so for this platform we should be fine to call `memalign` | |
193 | // (which is present in API level 9). Some helpful references could | |
194 | // possibly be chromium using memalign [1], attempts at documenting that | |
195 | // memalign + free is ok [2] [3], or the current source of chromium | |
196 | // which still uses memalign on android [4]. | |
197 | // | |
198 | // [1]: https://codereview.chromium.org/10796020/ | |
199 | // [2]: https://code.google.com/p/android/issues/detail?id=35391 | |
200 | // [3]: https://bugs.chromium.org/p/chromium/issues/detail?id=138579 | |
201 | // [4]: https://chromium.googlesource.com/chromium/src/base/+/master/ | |
202 | // /memory/aligned_memory.cc | |
83c7162d | 203 | libc::memalign(layout.align(), layout.size()) as *mut Opaque |
a7813a04 XL |
204 | } |
205 | ||
ea8adc8c | 206 | #[cfg(not(any(target_os = "android", target_os = "redox", target_os = "solaris")))] |
041b39d2 | 207 | #[inline] |
83c7162d | 208 | unsafe fn aligned_malloc(layout: &Layout) -> *mut Opaque { |
a7813a04 | 209 | let mut out = ptr::null_mut(); |
041b39d2 | 210 | let ret = libc::posix_memalign(&mut out, layout.align(), layout.size()); |
a7813a04 | 211 | if ret != 0 { |
83c7162d XL |
212 | // FIXME: use Opaque::null_mut https://github.com/rust-lang/rust/issues/49659 |
213 | 0 as *mut Opaque | |
a7813a04 | 214 | } else { |
83c7162d | 215 | out as *mut Opaque |
e9174d1e SL |
216 | } |
217 | } | |
e9174d1e SL |
218 | } |
219 | ||
3b2f2976 | 220 | #[cfg(windows)] |
92a42be0 | 221 | #[allow(bad_style)] |
041b39d2 | 222 | mod platform { |
e9174d1e | 223 | use MIN_ALIGN; |
3b2f2976 | 224 | use System; |
83c7162d | 225 | use core::alloc::{GlobalAlloc, Opaque, Layout}; |
e9174d1e | 226 | |
92a42be0 SL |
227 | type LPVOID = *mut u8; |
228 | type HANDLE = LPVOID; | |
229 | type SIZE_T = usize; | |
230 | type DWORD = u32; | |
231 | type BOOL = i32; | |
232 | ||
e9174d1e SL |
233 | extern "system" { |
234 | fn GetProcessHeap() -> HANDLE; | |
235 | fn HeapAlloc(hHeap: HANDLE, dwFlags: DWORD, dwBytes: SIZE_T) -> LPVOID; | |
b039eaaf | 236 | fn HeapReAlloc(hHeap: HANDLE, dwFlags: DWORD, lpMem: LPVOID, dwBytes: SIZE_T) -> LPVOID; |
e9174d1e | 237 | fn HeapFree(hHeap: HANDLE, dwFlags: DWORD, lpMem: LPVOID) -> BOOL; |
c30ab7b3 | 238 | fn GetLastError() -> DWORD; |
e9174d1e SL |
239 | } |
240 | ||
241 | #[repr(C)] | |
242 | struct Header(*mut u8); | |
243 | ||
cc61c64b | 244 | const HEAP_ZERO_MEMORY: DWORD = 0x00000008; |
e9174d1e SL |
245 | |
246 | unsafe fn get_header<'a>(ptr: *mut u8) -> &'a mut Header { | |
247 | &mut *(ptr as *mut Header).offset(-1) | |
248 | } | |
249 | ||
250 | unsafe fn align_ptr(ptr: *mut u8, align: usize) -> *mut u8 { | |
251 | let aligned = ptr.offset((align - (ptr as usize & (align - 1))) as isize); | |
252 | *get_header(aligned) = Header(ptr); | |
253 | aligned | |
254 | } | |
255 | ||
cc61c64b | 256 | #[inline] |
83c7162d | 257 | unsafe fn allocate_with_flags(layout: Layout, flags: DWORD) -> *mut Opaque { |
041b39d2 XL |
258 | let ptr = if layout.align() <= MIN_ALIGN { |
259 | HeapAlloc(GetProcessHeap(), flags, layout.size()) | |
e9174d1e | 260 | } else { |
041b39d2 XL |
261 | let size = layout.size() + layout.align(); |
262 | let ptr = HeapAlloc(GetProcessHeap(), flags, size); | |
b039eaaf | 263 | if ptr.is_null() { |
041b39d2 XL |
264 | ptr |
265 | } else { | |
266 | align_ptr(ptr, layout.align()) | |
b039eaaf | 267 | } |
041b39d2 | 268 | }; |
83c7162d | 269 | ptr as *mut Opaque |
e9174d1e SL |
270 | } |
271 | ||
041b39d2 | 272 | #[unstable(feature = "allocator_api", issue = "32838")] |
83c7162d | 273 | unsafe impl GlobalAlloc for System { |
041b39d2 | 274 | #[inline] |
83c7162d | 275 | unsafe fn alloc(&self, layout: Layout) -> *mut Opaque { |
041b39d2 XL |
276 | allocate_with_flags(layout, 0) |
277 | } | |
cc61c64b | 278 | |
041b39d2 | 279 | #[inline] |
83c7162d | 280 | unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut Opaque { |
041b39d2 XL |
281 | allocate_with_flags(layout, HEAP_ZERO_MEMORY) |
282 | } | |
cc61c64b | 283 | |
041b39d2 | 284 | #[inline] |
83c7162d | 285 | unsafe fn dealloc(&self, ptr: *mut Opaque, layout: Layout) { |
041b39d2 XL |
286 | if layout.align() <= MIN_ALIGN { |
287 | let err = HeapFree(GetProcessHeap(), 0, ptr as LPVOID); | |
288 | debug_assert!(err != 0, "Failed to free heap memory: {}", | |
289 | GetLastError()); | |
290 | } else { | |
83c7162d | 291 | let header = get_header(ptr as *mut u8); |
041b39d2 XL |
292 | let err = HeapFree(GetProcessHeap(), 0, header.0 as LPVOID); |
293 | debug_assert!(err != 0, "Failed to free heap memory: {}", | |
294 | GetLastError()); | |
b039eaaf | 295 | } |
e9174d1e | 296 | } |
e9174d1e | 297 | |
041b39d2 | 298 | #[inline] |
83c7162d XL |
299 | unsafe fn realloc(&self, ptr: *mut Opaque, layout: Layout, new_size: usize) -> *mut Opaque { |
300 | if layout.align() <= MIN_ALIGN { | |
301 | HeapReAlloc(GetProcessHeap(), 0, ptr as LPVOID, new_size) as *mut Opaque | |
041b39d2 | 302 | } else { |
83c7162d | 303 | self.realloc_fallback(ptr, layout, new_size) |
041b39d2 XL |
304 | } |
305 | } | |
e9174d1e SL |
306 | } |
307 | } | |
abe05a73 XL |
308 | |
309 | // This is an implementation of a global allocator on the wasm32 platform when | |
310 | // emscripten is not in use. In that situation there's no actual runtime for us | |
311 | // to lean on for allocation, so instead we provide our own! | |
312 | // | |
313 | // The wasm32 instruction set has two instructions for getting the current | |
314 | // amount of memory and growing the amount of memory. These instructions are the | |
315 | // foundation on which we're able to build an allocator, so we do so! Note that | |
316 | // the instructions are also pretty "global" and this is the "global" allocator | |
317 | // after all! | |
318 | // | |
319 | // The current allocator here is the `dlmalloc` crate which we've got included | |
320 | // in the rust-lang/rust repository as a submodule. The crate is a port of | |
321 | // dlmalloc.c from C to Rust and is basically just so we can have "pure Rust" | |
322 | // for now which is currently technically required (can't link with C yet). | |
323 | // | |
324 | // The crate itself provides a global allocator which on wasm has no | |
325 | // synchronization as there are no threads! | |
326 | #[cfg(all(target_arch = "wasm32", not(target_os = "emscripten")))] | |
327 | mod platform { | |
328 | extern crate dlmalloc; | |
329 | ||
83c7162d | 330 | use core::alloc::{GlobalAlloc, Layout, Opaque}; |
abe05a73 | 331 | use System; |
abe05a73 | 332 | |
83c7162d XL |
333 | // No need for synchronization here as wasm is currently single-threaded |
334 | static mut DLMALLOC: dlmalloc::Dlmalloc = dlmalloc::DLMALLOC_INIT; | |
abe05a73 | 335 | |
83c7162d XL |
336 | #[unstable(feature = "allocator_api", issue = "32838")] |
337 | unsafe impl GlobalAlloc for System { | |
abe05a73 | 338 | #[inline] |
83c7162d XL |
339 | unsafe fn alloc(&self, layout: Layout) -> *mut Opaque { |
340 | DLMALLOC.malloc(layout.size(), layout.align()) as *mut Opaque | |
abe05a73 XL |
341 | } |
342 | ||
343 | #[inline] | |
83c7162d XL |
344 | unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut Opaque { |
345 | DLMALLOC.calloc(layout.size(), layout.align()) as *mut Opaque | |
abe05a73 XL |
346 | } |
347 | ||
348 | #[inline] | |
83c7162d XL |
349 | unsafe fn dealloc(&self, ptr: *mut Opaque, layout: Layout) { |
350 | DLMALLOC.free(ptr as *mut u8, layout.size(), layout.align()) | |
abe05a73 XL |
351 | } |
352 | ||
353 | #[inline] | |
83c7162d XL |
354 | unsafe fn realloc(&self, ptr: *mut Opaque, layout: Layout, new_size: usize) -> *mut Opaque { |
355 | DLMALLOC.realloc(ptr as *mut u8, layout.size(), layout.align(), new_size) as *mut Opaque | |
abe05a73 XL |
356 | } |
357 | } | |
358 | } |