]>
Commit | Line | Data |
---|---|---|
f035d41b XL |
1 | // Copyright 2016 Amanieu d'Antras |
2 | // | |
3 | // Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or | |
4 | // http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or | |
5 | // http://opensource.org/licenses/MIT>, at your option. This file may not be | |
6 | // copied, modified, or distributed except according to those terms. | |
7 | ||
8 | use std::sync::atomic::AtomicUsize; | |
9 | ||
10 | // Extension trait to add lock elision primitives to atomic types | |
11 | pub trait AtomicElisionExt { | |
12 | type IntType; | |
13 | ||
14 | // Perform a compare_exchange and start a transaction | |
15 | fn elision_compare_exchange_acquire( | |
16 | &self, | |
17 | current: Self::IntType, | |
18 | new: Self::IntType, | |
19 | ) -> Result<Self::IntType, Self::IntType>; | |
20 | ||
21 | // Perform a fetch_sub and end a transaction | |
22 | fn elision_fetch_sub_release(&self, val: Self::IntType) -> Self::IntType; | |
23 | } | |
24 | ||
25 | // Indicates whether the target architecture supports lock elision | |
26 | #[inline] | |
27 | pub fn have_elision() -> bool { | |
28 | cfg!(all( | |
29 | feature = "nightly", | |
30 | any(target_arch = "x86", target_arch = "x86_64"), | |
31 | )) | |
32 | } | |
33 | ||
34 | // This implementation is never actually called because it is guarded by | |
35 | // have_elision(). | |
36 | #[cfg(not(all(feature = "nightly", any(target_arch = "x86", target_arch = "x86_64"))))] | |
37 | impl AtomicElisionExt for AtomicUsize { | |
38 | type IntType = usize; | |
39 | ||
40 | #[inline] | |
41 | fn elision_compare_exchange_acquire(&self, _: usize, _: usize) -> Result<usize, usize> { | |
42 | unreachable!(); | |
43 | } | |
44 | ||
45 | #[inline] | |
46 | fn elision_fetch_sub_release(&self, _: usize) -> usize { | |
47 | unreachable!(); | |
48 | } | |
49 | } | |
50 | ||
51 | #[cfg(all(feature = "nightly", any(target_arch = "x86", target_arch = "x86_64")))] | |
52 | impl AtomicElisionExt for AtomicUsize { | |
53 | type IntType = usize; | |
54 | ||
55 | #[cfg(target_pointer_width = "32")] | |
56 | #[inline] | |
57 | fn elision_compare_exchange_acquire(&self, current: usize, new: usize) -> Result<usize, usize> { | |
58 | unsafe { | |
59 | let prev: usize; | |
60 | llvm_asm!("xacquire; lock; cmpxchgl $2, $1" | |
61 | : "={eax}" (prev), "+*m" (self) | |
62 | : "r" (new), "{eax}" (current) | |
63 | : "memory" | |
64 | : "volatile"); | |
65 | if prev == current { | |
66 | Ok(prev) | |
67 | } else { | |
68 | Err(prev) | |
69 | } | |
70 | } | |
71 | } | |
72 | #[cfg(target_pointer_width = "64")] | |
73 | #[inline] | |
74 | fn elision_compare_exchange_acquire(&self, current: usize, new: usize) -> Result<usize, usize> { | |
75 | unsafe { | |
76 | let prev: usize; | |
77 | llvm_asm!("xacquire; lock; cmpxchgq $2, $1" | |
78 | : "={rax}" (prev), "+*m" (self) | |
79 | : "r" (new), "{rax}" (current) | |
80 | : "memory" | |
81 | : "volatile"); | |
82 | if prev == current { | |
83 | Ok(prev) | |
84 | } else { | |
85 | Err(prev) | |
86 | } | |
87 | } | |
88 | } | |
89 | ||
90 | #[cfg(target_pointer_width = "32")] | |
91 | #[inline] | |
92 | fn elision_fetch_sub_release(&self, val: usize) -> usize { | |
93 | unsafe { | |
94 | let prev: usize; | |
95 | llvm_asm!("xrelease; lock; xaddl $2, $1" | |
96 | : "=r" (prev), "+*m" (self) | |
97 | : "0" (val.wrapping_neg()) | |
98 | : "memory" | |
99 | : "volatile"); | |
100 | prev | |
101 | } | |
102 | } | |
103 | #[cfg(target_pointer_width = "64")] | |
104 | #[inline] | |
105 | fn elision_fetch_sub_release(&self, val: usize) -> usize { | |
106 | unsafe { | |
107 | let prev: usize; | |
108 | llvm_asm!("xrelease; lock; xaddq $2, $1" | |
109 | : "=r" (prev), "+*m" (self) | |
110 | : "0" (val.wrapping_neg()) | |
111 | : "memory" | |
112 | : "volatile"); | |
113 | prev | |
114 | } | |
115 | } | |
116 | } |