src/liballoc_system/lib.rs

   1 // Copyright 2015 The Rust Project Developers. See the COPYRIGHT
   2 // file at the top-level directory of this distribution and at
   3 // http://rust-lang.org/COPYRIGHT.
   4 //
   5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 // option. This file may not be copied, modified, or distributed
   9 // except according to those terms.
  10
  11 #![no_std]
  12 #![allow(unused_attributes)]
  13 #![deny(warnings)]
  14 #![unstable(feature = "alloc_system",
  15             reason = "this library is unlikely to be stabilized in its current \
  16                       form or name",
  17             issue = "32838")]
  18 #![feature(global_allocator)]
  19 #![feature(allocator_api)]
  20 #![feature(alloc)]
  21 #![feature(core_intrinsics)]
  22 #![feature(staged_api)]
  23 #![feature(rustc_attrs)]
  24 #![cfg_attr(any(unix, target_os = "redox"), feature(libc))]
  25 #![rustc_alloc_kind = "lib"]
  26
  27 // The minimum alignment guaranteed by the architecture. This value is used to
  28 // add fast paths for low alignment values. In practice, the alignment is a
  29 // constant at the call site and the branch will be optimized out.
  30 #[cfg(all(any(target_arch = "x86",
  31               target_arch = "arm",
  32               target_arch = "mips",
  33               target_arch = "powerpc",
  34               target_arch = "powerpc64",
  35               target_arch = "asmjs",
  36               target_arch = "wasm32")))]
  37 #[allow(dead_code)]
  38 const MIN_ALIGN: usize = 8;
  39 #[cfg(all(any(target_arch = "x86_64",
  40               target_arch = "aarch64",
  41               target_arch = "mips64",
  42               target_arch = "s390x",
  43               target_arch = "sparc64")))]
  44 #[allow(dead_code)]
  45 const MIN_ALIGN: usize = 16;
  46
  47 extern crate alloc;
  48
  49 use self::alloc::heap::{Alloc, AllocErr, Layout, Excess, CannotReallocInPlace};
  50
  51 #[unstable(feature = "allocator_api", issue = "32838")]
  52 pub struct System;
  53
  54 #[unstable(feature = "allocator_api", issue = "32838")]
  55 unsafe impl Alloc for System {
  56     #[inline]
  57     unsafe fn alloc(&mut self, layout: Layout) -> Result<*mut u8, AllocErr> {
  58         (&*self).alloc(layout)
  59     }
  60
  61     #[inline]
  62     unsafe fn alloc_zeroed(&mut self, layout: Layout)
  63         -> Result<*mut u8, AllocErr>
  64     {
  65         (&*self).alloc_zeroed(layout)
  66     }
  67
  68     #[inline]
  69     unsafe fn dealloc(&mut self, ptr: *mut u8, layout: Layout) {
  70         (&*self).dealloc(ptr, layout)
  71     }
  72
  73     #[inline]
  74     unsafe fn realloc(&mut self,
  75                       ptr: *mut u8,
  76                       old_layout: Layout,
  77                       new_layout: Layout) -> Result<*mut u8, AllocErr> {
  78         (&*self).realloc(ptr, old_layout, new_layout)
  79     }
  80
  81     fn oom(&mut self, err: AllocErr) -> ! {
  82         (&*self).oom(err)
  83     }
  84
  85     #[inline]
  86     fn usable_size(&self, layout: &Layout) -> (usize, usize) {
  87         (&self).usable_size(layout)
  88     }
  89
  90     #[inline]
  91     unsafe fn alloc_excess(&mut self, layout: Layout) -> Result<Excess, AllocErr> {
  92         (&*self).alloc_excess(layout)
  93     }
  94
  95     #[inline]
  96     unsafe fn realloc_excess(&mut self,
  97                              ptr: *mut u8,
  98                              layout: Layout,
  99                              new_layout: Layout) -> Result<Excess, AllocErr> {
 100         (&*self).realloc_excess(ptr, layout, new_layout)
 101     }
 102
 103     #[inline]
 104     unsafe fn grow_in_place(&mut self,
 105                             ptr: *mut u8,
 106                             layout: Layout,
 107                             new_layout: Layout) -> Result<(), CannotReallocInPlace> {
 108         (&*self).grow_in_place(ptr, layout, new_layout)
 109     }
 110
 111     #[inline]
 112     unsafe fn shrink_in_place(&mut self,
 113                               ptr: *mut u8,
 114                               layout: Layout,
 115                               new_layout: Layout) -> Result<(), CannotReallocInPlace> {
 116         (&*self).shrink_in_place(ptr, layout, new_layout)
 117     }
 118 }
 119
 120 #[cfg(any(unix, target_os = "redox"))]
 121 mod platform {
 122     extern crate libc;
 123
 124     use core::cmp;
 125     use core::ptr;
 126
 127     use MIN_ALIGN;
 128     use System;
 129     use alloc::heap::{Alloc, AllocErr, Layout};
 130
 131     #[unstable(feature = "allocator_api", issue = "32838")]
 132     unsafe impl<'a> Alloc for &'a System {
 133         #[inline]
 134         unsafe fn alloc(&mut self, layout: Layout) -> Result<*mut u8, AllocErr> {
 135             let ptr = if layout.align() <= MIN_ALIGN {
 136                 libc::malloc(layout.size()) as *mut u8
 137             } else {
 138                 aligned_malloc(&layout)
 139             };
 140             if !ptr.is_null() {
 141                 Ok(ptr)
 142             } else {
 143                 Err(AllocErr::Exhausted { request: layout })
 144             }
 145         }
 146
 147         #[inline]
 148         unsafe fn alloc_zeroed(&mut self, layout: Layout)
 149             -> Result<*mut u8, AllocErr>
 150         {
 151             if layout.align() <= MIN_ALIGN {
 152                 let ptr = libc::calloc(layout.size(), 1) as *mut u8;
 153                 if !ptr.is_null() {
 154                     Ok(ptr)
 155                 } else {
 156                     Err(AllocErr::Exhausted { request: layout })
 157                 }
 158             } else {
 159                 let ret = self.alloc(layout.clone());
 160                 if let Ok(ptr) = ret {
 161                     ptr::write_bytes(ptr, 0, layout.size());
 162                 }
 163                 ret
 164             }
 165         }
 166
 167         #[inline]
 168         unsafe fn dealloc(&mut self, ptr: *mut u8, _layout: Layout) {
 169             libc::free(ptr as *mut libc::c_void)
 170         }
 171
 172         #[inline]
 173         unsafe fn realloc(&mut self,
 174                           ptr: *mut u8,
 175                           old_layout: Layout,
 176                           new_layout: Layout) -> Result<*mut u8, AllocErr> {
 177             if old_layout.align() != new_layout.align() {
 178                 return Err(AllocErr::Unsupported {
 179                     details: "cannot change alignment on `realloc`",
 180                 })
 181             }
 182
 183             if new_layout.align() <= MIN_ALIGN {
 184                 let ptr = libc::realloc(ptr as *mut libc::c_void, new_layout.size());
 185                 if !ptr.is_null() {
 186                     Ok(ptr as *mut u8)
 187                 } else {
 188                     Err(AllocErr::Exhausted { request: new_layout })
 189                 }
 190             } else {
 191                 let res = self.alloc(new_layout.clone());
 192                 if let Ok(new_ptr) = res {
 193                     let size = cmp::min(old_layout.size(), new_layout.size());
 194                     ptr::copy_nonoverlapping(ptr, new_ptr, size);
 195                     self.dealloc(ptr, old_layout);
 196                 }
 197                 res
 198             }
 199         }
 200
 201         fn oom(&mut self, err: AllocErr) -> ! {
 202             use core::fmt::{self, Write};
 203
 204             // Print a message to stderr before aborting to assist with
 205             // debugging. It is critical that this code does not allocate any
 206             // memory since we are in an OOM situation. Any errors are ignored
 207             // while printing since there's nothing we can do about them and we
 208             // are about to exit anyways.
 209             drop(writeln!(Stderr, "fatal runtime error: {}", err));
 210             unsafe {
 211                 ::core::intrinsics::abort();
 212             }
 213
 214             struct Stderr;
 215
 216             impl Write for Stderr {
 217                 fn write_str(&mut self, s: &str) -> fmt::Result {
 218                     unsafe {
 219                         libc::write(libc::STDERR_FILENO,
 220                                     s.as_ptr() as *const libc::c_void,
 221                                     s.len());
 222                     }
 223                     Ok(())
 224                 }
 225             }
 226         }
 227     }
 228
 229     #[cfg(any(target_os = "android", target_os = "redox", target_os = "solaris"))]
 230     #[inline]
 231     unsafe fn aligned_malloc(layout: &Layout) -> *mut u8 {
 232         // On android we currently target API level 9 which unfortunately
 233         // doesn't have the `posix_memalign` API used below. Instead we use
 234         // `memalign`, but this unfortunately has the property on some systems
 235         // where the memory returned cannot be deallocated by `free`!
 236         //
 237         // Upon closer inspection, however, this appears to work just fine with
 238         // Android, so for this platform we should be fine to call `memalign`
 239         // (which is present in API level 9). Some helpful references could
 240         // possibly be chromium using memalign [1], attempts at documenting that
 241         // memalign + free is ok [2] [3], or the current source of chromium
 242         // which still uses memalign on android [4].
 243         //
 244         // [1]: https://codereview.chromium.org/10796020/
 245         // [2]: https://code.google.com/p/android/issues/detail?id=35391
 246         // [3]: https://bugs.chromium.org/p/chromium/issues/detail?id=138579
 247         // [4]: https://chromium.googlesource.com/chromium/src/base/+/master/
 248         //                                       /memory/aligned_memory.cc
 249         libc::memalign(layout.align(), layout.size()) as *mut u8
 250     }
 251
 252     #[cfg(not(any(target_os = "android", target_os = "redox", target_os = "solaris")))]
 253     #[inline]
 254     unsafe fn aligned_malloc(layout: &Layout) -> *mut u8 {
 255         let mut out = ptr::null_mut();
 256         let ret = libc::posix_memalign(&mut out, layout.align(), layout.size());
 257         if ret != 0 {
 258             ptr::null_mut()
 259         } else {
 260             out as *mut u8
 261         }
 262     }
 263 }
 264
 265 #[cfg(windows)]
 266 #[allow(bad_style)]
 267 mod platform {
 268     use core::cmp;
 269     use core::ptr;
 270
 271     use MIN_ALIGN;
 272     use System;
 273     use alloc::heap::{Alloc, AllocErr, Layout, CannotReallocInPlace};
 274
 275     type LPVOID = *mut u8;
 276     type HANDLE = LPVOID;
 277     type SIZE_T = usize;
 278     type DWORD = u32;
 279     type BOOL = i32;
 280     type LPDWORD = *mut DWORD;
 281     type LPOVERLAPPED = *mut u8;
 282
 283     const STD_ERROR_HANDLE: DWORD = -12i32 as DWORD;
 284
 285     extern "system" {
 286         fn GetProcessHeap() -> HANDLE;
 287         fn HeapAlloc(hHeap: HANDLE, dwFlags: DWORD, dwBytes: SIZE_T) -> LPVOID;
 288         fn HeapReAlloc(hHeap: HANDLE, dwFlags: DWORD, lpMem: LPVOID, dwBytes: SIZE_T) -> LPVOID;
 289         fn HeapFree(hHeap: HANDLE, dwFlags: DWORD, lpMem: LPVOID) -> BOOL;
 290         fn GetLastError() -> DWORD;
 291         fn WriteFile(hFile: HANDLE,
 292                      lpBuffer: LPVOID,
 293                      nNumberOfBytesToWrite: DWORD,
 294                      lpNumberOfBytesWritten: LPDWORD,
 295                      lpOverlapped: LPOVERLAPPED)
 296                      -> BOOL;
 297         fn GetStdHandle(which: DWORD) -> HANDLE;
 298     }
 299
 300     #[repr(C)]
 301     struct Header(*mut u8);
 302
 303     const HEAP_ZERO_MEMORY: DWORD = 0x00000008;
 304     const HEAP_REALLOC_IN_PLACE_ONLY: DWORD = 0x00000010;
 305
 306     unsafe fn get_header<'a>(ptr: *mut u8) -> &'a mut Header {
 307         &mut *(ptr as *mut Header).offset(-1)
 308     }
 309
 310     unsafe fn align_ptr(ptr: *mut u8, align: usize) -> *mut u8 {
 311         let aligned = ptr.offset((align - (ptr as usize & (align - 1))) as isize);
 312         *get_header(aligned) = Header(ptr);
 313         aligned
 314     }
 315
 316     #[inline]
 317     unsafe fn allocate_with_flags(layout: Layout, flags: DWORD)
 318         -> Result<*mut u8, AllocErr>
 319     {
 320         let ptr = if layout.align() <= MIN_ALIGN {
 321             HeapAlloc(GetProcessHeap(), flags, layout.size())
 322         } else {
 323             let size = layout.size() + layout.align();
 324             let ptr = HeapAlloc(GetProcessHeap(), flags, size);
 325             if ptr.is_null() {
 326                 ptr
 327             } else {
 328                 align_ptr(ptr, layout.align())
 329             }
 330         };
 331         if ptr.is_null() {
 332             Err(AllocErr::Exhausted { request: layout })
 333         } else {
 334             Ok(ptr as *mut u8)
 335         }
 336     }
 337
 338     #[unstable(feature = "allocator_api", issue = "32838")]
 339     unsafe impl<'a> Alloc for &'a System {
 340         #[inline]
 341         unsafe fn alloc(&mut self, layout: Layout) -> Result<*mut u8, AllocErr> {
 342             allocate_with_flags(layout, 0)
 343         }
 344
 345         #[inline]
 346         unsafe fn alloc_zeroed(&mut self, layout: Layout)
 347             -> Result<*mut u8, AllocErr>
 348         {
 349             allocate_with_flags(layout, HEAP_ZERO_MEMORY)
 350         }
 351
 352         #[inline]
 353         unsafe fn dealloc(&mut self, ptr: *mut u8, layout: Layout) {
 354             if layout.align() <= MIN_ALIGN {
 355                 let err = HeapFree(GetProcessHeap(), 0, ptr as LPVOID);
 356                 debug_assert!(err != 0, "Failed to free heap memory: {}",
 357                               GetLastError());
 358             } else {
 359                 let header = get_header(ptr);
 360                 let err = HeapFree(GetProcessHeap(), 0, header.0 as LPVOID);
 361                 debug_assert!(err != 0, "Failed to free heap memory: {}",
 362                               GetLastError());
 363             }
 364         }
 365
 366         #[inline]
 367         unsafe fn realloc(&mut self,
 368                           ptr: *mut u8,
 369                           old_layout: Layout,
 370                           new_layout: Layout) -> Result<*mut u8, AllocErr> {
 371             if old_layout.align() != new_layout.align() {
 372                 return Err(AllocErr::Unsupported {
 373                     details: "cannot change alignment on `realloc`",
 374                 })
 375             }
 376
 377             if new_layout.align() <= MIN_ALIGN {
 378                 let ptr = HeapReAlloc(GetProcessHeap(),
 379                                       0,
 380                                       ptr as LPVOID,
 381                                       new_layout.size());
 382                 if !ptr.is_null() {
 383                     Ok(ptr as *mut u8)
 384                 } else {
 385                     Err(AllocErr::Exhausted { request: new_layout })
 386                 }
 387             } else {
 388                 let res = self.alloc(new_layout.clone());
 389                 if let Ok(new_ptr) = res {
 390                     let size = cmp::min(old_layout.size(), new_layout.size());
 391                     ptr::copy_nonoverlapping(ptr, new_ptr, size);
 392                     self.dealloc(ptr, old_layout);
 393                 }
 394                 res
 395             }
 396         }
 397
 398         #[inline]
 399         unsafe fn grow_in_place(&mut self,
 400                                 ptr: *mut u8,
 401                                 layout: Layout,
 402                                 new_layout: Layout) -> Result<(), CannotReallocInPlace> {
 403             self.shrink_in_place(ptr, layout, new_layout)
 404         }
 405
 406         #[inline]
 407         unsafe fn shrink_in_place(&mut self,
 408                                   ptr: *mut u8,
 409                                   old_layout: Layout,
 410                                   new_layout: Layout) -> Result<(), CannotReallocInPlace> {
 411             if old_layout.align() != new_layout.align() {
 412                 return Err(CannotReallocInPlace)
 413             }
 414
 415             let new = if new_layout.align() <= MIN_ALIGN {
 416                 HeapReAlloc(GetProcessHeap(),
 417                             HEAP_REALLOC_IN_PLACE_ONLY,
 418                             ptr as LPVOID,
 419                             new_layout.size())
 420             } else {
 421                 let header = get_header(ptr);
 422                 HeapReAlloc(GetProcessHeap(),
 423                             HEAP_REALLOC_IN_PLACE_ONLY,
 424                             header.0 as LPVOID,
 425                             new_layout.size() + new_layout.align())
 426             };
 427             if new.is_null() {
 428                 Err(CannotReallocInPlace)
 429             } else {
 430                 Ok(())
 431             }
 432         }
 433
 434         fn oom(&mut self, err: AllocErr) -> ! {
 435             use core::fmt::{self, Write};
 436
 437             // Same as with unix we ignore all errors here
 438             drop(writeln!(Stderr, "fatal runtime error: {}", err));
 439             unsafe {
 440                 ::core::intrinsics::abort();
 441             }
 442
 443             struct Stderr;
 444
 445             impl Write for Stderr {
 446                 fn write_str(&mut self, s: &str) -> fmt::Result {
 447                     unsafe {
 448                         // WriteFile silently fails if it is passed an invalid
 449                         // handle, so there is no need to check the result of
 450                         // GetStdHandle.
 451                         WriteFile(GetStdHandle(STD_ERROR_HANDLE),
 452                                   s.as_ptr() as LPVOID,
 453                                   s.len() as DWORD,
 454                                   ptr::null_mut(),
 455                                   ptr::null_mut());
 456                     }
 457                     Ok(())
 458                 }
 459             }
 460         }
 461     }
 462 }
 463
 464 // This is an implementation of a global allocator on the wasm32 platform when
 465 // emscripten is not in use. In that situation there's no actual runtime for us
 466 // to lean on for allocation, so instead we provide our own!
 467 //
 468 // The wasm32 instruction set has two instructions for getting the current
 469 // amount of memory and growing the amount of memory. These instructions are the
 470 // foundation on which we're able to build an allocator, so we do so! Note that
 471 // the instructions are also pretty "global" and this is the "global" allocator
 472 // after all!
 473 //
 474 // The current allocator here is the `dlmalloc` crate which we've got included
 475 // in the rust-lang/rust repository as a submodule. The crate is a port of
 476 // dlmalloc.c from C to Rust and is basically just so we can have "pure Rust"
 477 // for now which is currently technically required (can't link with C yet).
 478 //
 479 // The crate itself provides a global allocator which on wasm has no
 480 // synchronization as there are no threads!
 481 #[cfg(all(target_arch = "wasm32", not(target_os = "emscripten")))]
 482 mod platform {
 483     extern crate dlmalloc;
 484
 485     use alloc::heap::{Alloc, AllocErr, Layout, Excess, CannotReallocInPlace};
 486     use System;
 487     use self::dlmalloc::GlobalDlmalloc;
 488
 489     #[unstable(feature = "allocator_api", issue = "32838")]
 490     unsafe impl<'a> Alloc for &'a System {
 491         #[inline]
 492         unsafe fn alloc(&mut self, layout: Layout) -> Result<*mut u8, AllocErr> {
 493             GlobalDlmalloc.alloc(layout)
 494         }
 495
 496         #[inline]
 497         unsafe fn alloc_zeroed(&mut self, layout: Layout)
 498             -> Result<*mut u8, AllocErr>
 499         {
 500             GlobalDlmalloc.alloc_zeroed(layout)
 501         }
 502
 503         #[inline]
 504         unsafe fn dealloc(&mut self, ptr: *mut u8, layout: Layout) {
 505             GlobalDlmalloc.dealloc(ptr, layout)
 506         }
 507
 508         #[inline]
 509         unsafe fn realloc(&mut self,
 510                           ptr: *mut u8,
 511                           old_layout: Layout,
 512                           new_layout: Layout) -> Result<*mut u8, AllocErr> {
 513             GlobalDlmalloc.realloc(ptr, old_layout, new_layout)
 514         }
 515
 516         #[inline]
 517         fn usable_size(&self, layout: &Layout) -> (usize, usize) {
 518             GlobalDlmalloc.usable_size(layout)
 519         }
 520
 521         #[inline]
 522         unsafe fn alloc_excess(&mut self, layout: Layout) -> Result<Excess, AllocErr> {
 523             GlobalDlmalloc.alloc_excess(layout)
 524         }
 525
 526         #[inline]
 527         unsafe fn realloc_excess(&mut self,
 528                                  ptr: *mut u8,
 529                                  layout: Layout,
 530                                  new_layout: Layout) -> Result<Excess, AllocErr> {
 531             GlobalDlmalloc.realloc_excess(ptr, layout, new_layout)
 532         }
 533
 534         #[inline]
 535         unsafe fn grow_in_place(&mut self,
 536                                 ptr: *mut u8,
 537                                 layout: Layout,
 538                                 new_layout: Layout) -> Result<(), CannotReallocInPlace> {
 539             GlobalDlmalloc.grow_in_place(ptr, layout, new_layout)
 540         }
 541
 542         #[inline]
 543         unsafe fn shrink_in_place(&mut self,
 544                                   ptr: *mut u8,
 545                                   layout: Layout,
 546                                   new_layout: Layout) -> Result<(), CannotReallocInPlace> {
 547             GlobalDlmalloc.shrink_in_place(ptr, layout, new_layout)
 548         }
 549     }
 550 }