]>
Commit | Line | Data |
---|---|---|
0531ce1d XL |
1 | //! Caches run-time feature detection so that it only needs to be computed |
2 | //! once. | |
3 | ||
4 | #![allow(dead_code)] // not used on all platforms | |
5 | ||
cdc7bbd5 | 6 | use core::sync::atomic::Ordering; |
0531ce1d | 7 | |
cdc7bbd5 | 8 | use core::sync::atomic::AtomicUsize; |
0531ce1d XL |
9 | |
10 | /// Sets the `bit` of `x`. | |
8faf50e0 XL |
11 | #[inline] |
12 | const fn set_bit(x: u64, bit: u32) -> u64 { | |
0531ce1d XL |
13 | x | 1 << bit |
14 | } | |
15 | ||
16 | /// Tests the `bit` of `x`. | |
8faf50e0 XL |
17 | #[inline] |
18 | const fn test_bit(x: u64, bit: u32) -> bool { | |
0531ce1d XL |
19 | x & (1 << bit) != 0 |
20 | } | |
21 | ||
74b04a01 XL |
22 | /// Unset the `bit of `x`. |
23 | #[inline] | |
24 | const fn unset_bit(x: u64, bit: u32) -> u64 { | |
25 | x & !(1 << bit) | |
26 | } | |
27 | ||
0531ce1d | 28 | /// Maximum number of features that can be cached. |
74b04a01 | 29 | const CACHE_CAPACITY: u32 = 62; |
0531ce1d XL |
30 | |
31 | /// This type is used to initialize the cache | |
32 | #[derive(Copy, Clone)] | |
8faf50e0 | 33 | pub(crate) struct Initializer(u64); |
0531ce1d | 34 | |
48663c56 | 35 | #[allow(clippy::use_self)] |
0531ce1d XL |
36 | impl Default for Initializer { |
37 | fn default() -> Self { | |
38 | Initializer(0) | |
39 | } | |
40 | } | |
41 | ||
74b04a01 XL |
42 | // NOTE: the `debug_assert!` would catch that we do not add more Features than |
43 | // the one fitting our cache. | |
0531ce1d XL |
44 | impl Initializer { |
45 | /// Tests the `bit` of the cache. | |
8faf50e0 | 46 | #[inline] |
0731742a | 47 | pub(crate) fn test(self, bit: u32) -> bool { |
0531ce1d XL |
48 | debug_assert!( |
49 | bit < CACHE_CAPACITY, | |
50 | "too many features, time to increase the cache size!" | |
51 | ); | |
52 | test_bit(self.0, bit) | |
53 | } | |
54 | ||
55 | /// Sets the `bit` of the cache. | |
8faf50e0 XL |
56 | #[inline] |
57 | pub(crate) fn set(&mut self, bit: u32) { | |
0531ce1d XL |
58 | debug_assert!( |
59 | bit < CACHE_CAPACITY, | |
60 | "too many features, time to increase the cache size!" | |
61 | ); | |
62 | let v = self.0; | |
63 | self.0 = set_bit(v, bit); | |
64 | } | |
74b04a01 XL |
65 | |
66 | /// Unsets the `bit` of the cache. | |
67 | #[inline] | |
68 | pub(crate) fn unset(&mut self, bit: u32) { | |
69 | debug_assert!( | |
70 | bit < CACHE_CAPACITY, | |
71 | "too many features, time to increase the cache size!" | |
72 | ); | |
73 | let v = self.0; | |
74 | self.0 = unset_bit(v, bit); | |
75 | } | |
0531ce1d XL |
76 | } |
77 | ||
78 | /// This global variable is a cache of the features supported by the CPU. | |
74b04a01 XL |
79 | // Note: on x64, we only use the first slot |
80 | static CACHE: [Cache; 2] = [Cache::uninitialized(), Cache::uninitialized()]; | |
0531ce1d | 81 | |
1b1a35ee | 82 | /// Feature cache with capacity for `size_of::<usize::MAX>() * 8 - 1` features. |
0531ce1d | 83 | /// |
1b1a35ee XL |
84 | /// Note: 0 is used to represent an uninitialized cache, and (at least) the most |
85 | /// significant bit is set on any cache which has been initialized. | |
74b04a01 | 86 | /// |
1b1a35ee XL |
87 | /// Note: we use `Relaxed` atomic operations, because we are only interested in |
88 | /// the effects of operations on a single memory location. That is, we only need | |
89 | /// "modification order", and not the full-blown "happens before". | |
74b04a01 | 90 | struct Cache(AtomicUsize); |
0531ce1d | 91 | |
0531ce1d | 92 | impl Cache { |
74b04a01 XL |
93 | const CAPACITY: u32 = (core::mem::size_of::<usize>() * 8 - 1) as u32; |
94 | const MASK: usize = (1 << Cache::CAPACITY) - 1; | |
1b1a35ee | 95 | const INITIALIZED_BIT: usize = 1usize << Cache::CAPACITY; |
74b04a01 | 96 | |
0531ce1d | 97 | /// Creates an uninitialized cache. |
48663c56 | 98 | #[allow(clippy::declare_interior_mutable_const)] |
0531ce1d | 99 | const fn uninitialized() -> Self { |
1b1a35ee | 100 | Cache(AtomicUsize::new(0)) |
0531ce1d XL |
101 | } |
102 | ||
1b1a35ee | 103 | /// Is the `bit` in the cache set? Returns `None` if the cache has not been initialized. |
8faf50e0 | 104 | #[inline] |
1b1a35ee XL |
105 | pub(crate) fn test(&self, bit: u32) -> Option<bool> { |
106 | let cached = self.0.load(Ordering::Relaxed); | |
107 | if cached == 0 { | |
108 | None | |
109 | } else { | |
110 | Some(test_bit(cached as u64, bit)) | |
111 | } | |
0531ce1d XL |
112 | } |
113 | ||
114 | /// Initializes the cache. | |
8faf50e0 | 115 | #[inline] |
1b1a35ee XL |
116 | fn initialize(&self, value: usize) -> usize { |
117 | debug_assert_eq!((value & !Cache::MASK), 0); | |
118 | self.0 | |
119 | .store(value | Cache::INITIALIZED_BIT, Ordering::Relaxed); | |
120 | value | |
0531ce1d XL |
121 | } |
122 | } | |
123 | ||
74b04a01 XL |
124 | cfg_if::cfg_if! { |
125 | if #[cfg(feature = "std_detect_env_override")] { | |
1b1a35ee XL |
126 | #[inline] |
127 | fn initialize(mut value: Initializer) -> Initializer { | |
cdc7bbd5 XL |
128 | let env = unsafe { |
129 | libc::getenv(b"RUST_STD_DETECT_UNSTABLE\0".as_ptr() as *const libc::c_char) | |
130 | }; | |
131 | if !env.is_null() { | |
132 | let len = unsafe { libc::strlen(env) }; | |
133 | let env = unsafe { core::slice::from_raw_parts(env as *const u8, len) }; | |
134 | if let Ok(disable) = core::str::from_utf8(env) { | |
135 | for v in disable.split(" ") { | |
136 | let _ = super::Feature::from_str(v).map(|v| value.unset(v as u32)); | |
137 | } | |
74b04a01 XL |
138 | } |
139 | } | |
140 | do_initialize(value); | |
1b1a35ee | 141 | value |
74b04a01 XL |
142 | } |
143 | } else { | |
144 | #[inline] | |
1b1a35ee | 145 | fn initialize(value: Initializer) -> Initializer { |
74b04a01 | 146 | do_initialize(value); |
1b1a35ee | 147 | value |
0531ce1d XL |
148 | } |
149 | } | |
74b04a01 | 150 | } |
0531ce1d | 151 | |
74b04a01 XL |
152 | #[inline] |
153 | fn do_initialize(value: Initializer) { | |
154 | CACHE[0].initialize((value.0) as usize & Cache::MASK); | |
155 | CACHE[1].initialize((value.0 >> Cache::CAPACITY) as usize & Cache::MASK); | |
0531ce1d XL |
156 | } |
157 | ||
1b1a35ee XL |
158 | // We only have to detect features once, and it's fairly costly, so hint to LLVM |
159 | // that it should assume that cache hits are more common than misses (which is | |
160 | // the point of caching). It's possibly unfortunate that this function needs to | |
161 | // reach across modules like this to call `os::detect_features`, but it produces | |
162 | // the best code out of several attempted variants. | |
163 | // | |
164 | // The `Initializer` that the cache was initialized with is returned, so that | |
165 | // the caller can call `test()` on it without having to load the value from the | |
166 | // cache again. | |
167 | #[cold] | |
168 | fn detect_and_initialize() -> Initializer { | |
169 | initialize(super::os::detect_features()) | |
170 | } | |
171 | ||
532ac7d7 | 172 | /// Tests the `bit` of the storage. If the storage has not been initialized, |
1b1a35ee | 173 | /// initializes it with the result of `os::detect_features()`. |
0531ce1d XL |
174 | /// |
175 | /// On its first invocation, it detects the CPU features and caches them in the | |
83c7162d | 176 | /// `CACHE` global variable as an `AtomicU64`. |
0531ce1d XL |
177 | /// |
178 | /// It uses the `Feature` variant to index into this variable as a bitset. If | |
179 | /// the bit is set, the feature is enabled, and otherwise it is disabled. | |
74b04a01 XL |
180 | /// |
181 | /// If the feature `std_detect_env_override` is enabled looks for the env | |
182 | /// variable `RUST_STD_DETECT_UNSTABLE` and uses its its content to disable | |
183 | /// Features that would had been otherwise detected. | |
8faf50e0 | 184 | #[inline] |
1b1a35ee XL |
185 | pub(crate) fn test(bit: u32) -> bool { |
186 | let (relative_bit, idx) = if bit < Cache::CAPACITY { | |
74b04a01 XL |
187 | (bit, 0) |
188 | } else { | |
189 | (bit - Cache::CAPACITY, 1) | |
190 | }; | |
1b1a35ee XL |
191 | CACHE[idx] |
192 | .test(relative_bit) | |
193 | .unwrap_or_else(|| detect_and_initialize().test(bit)) | |
0531ce1d | 194 | } |