]>
Commit | Line | Data |
---|---|---|
1a4d82fc | 1 | //! Thread local storage |
c34b1796 | 2 | |
dfeec247 | 3 | #![unstable(feature = "thread_local_internals", issue = "none")] |
1a4d82fc | 4 | |
1b1a35ee XL |
5 | #[cfg(all(test, not(target_os = "emscripten")))] |
6 | mod tests; | |
7 | ||
8 | #[cfg(test)] | |
9 | mod dynamic_tests; | |
10 | ||
416331ca | 11 | use crate::error::Error; |
532ac7d7 | 12 | use crate::fmt; |
1a4d82fc JJ |
13 | |
14 | /// A thread local storage key which owns its contents. | |
15 | /// | |
16 | /// This key uses the fastest possible implementation available to it for the | |
7cac9316 XL |
17 | /// target platform. It is instantiated with the [`thread_local!`] macro and the |
18 | /// primary method is the [`with`] method. | |
1a4d82fc | 19 | /// |
7cac9316 | 20 | /// The [`with`] method yields a reference to the contained value which cannot be |
bd371182 | 21 | /// sent across threads or escape the given closure. |
1a4d82fc JJ |
22 | /// |
23 | /// # Initialization and Destruction | |
24 | /// | |
7cac9316 XL |
25 | /// Initialization is dynamically performed on the first call to [`with`] |
26 | /// within a thread, and values that implement [`Drop`] get destructed when a | |
8bb4bdeb | 27 | /// thread exits. Some caveats apply, which are explained below. |
1a4d82fc | 28 | /// |
ea8adc8c XL |
29 | /// A `LocalKey`'s initializer cannot recursively depend on itself, and using |
30 | /// a `LocalKey` in this way will cause the initializer to infinitely recurse | |
31 | /// on the first call to `with`. | |
32 | /// | |
c34b1796 | 33 | /// # Examples |
1a4d82fc JJ |
34 | /// |
35 | /// ``` | |
36 | /// use std::cell::RefCell; | |
85aaf69f | 37 | /// use std::thread; |
1a4d82fc | 38 | /// |
c34b1796 | 39 | /// thread_local!(static FOO: RefCell<u32> = RefCell::new(1)); |
1a4d82fc JJ |
40 | /// |
41 | /// FOO.with(|f| { | |
42 | /// assert_eq!(*f.borrow(), 1); | |
43 | /// *f.borrow_mut() = 2; | |
44 | /// }); | |
45 | /// | |
46 | /// // each thread starts out with the initial value of 1 | |
532ac7d7 | 47 | /// let t = thread::spawn(move|| { |
1a4d82fc JJ |
48 | /// FOO.with(|f| { |
49 | /// assert_eq!(*f.borrow(), 1); | |
50 | /// *f.borrow_mut() = 3; | |
51 | /// }); | |
52 | /// }); | |
53 | /// | |
532ac7d7 XL |
54 | /// // wait for the thread to complete and bail out on panic |
55 | /// t.join().unwrap(); | |
56 | /// | |
1a4d82fc JJ |
57 | /// // we retain our original value of 2 despite the child thread |
58 | /// FOO.with(|f| { | |
59 | /// assert_eq!(*f.borrow(), 2); | |
60 | /// }); | |
61 | /// ``` | |
7453a54e SL |
62 | /// |
63 | /// # Platform-specific behavior | |
64 | /// | |
65 | /// Note that a "best effort" is made to ensure that destructors for types | |
a7813a04 | 66 | /// stored in thread local storage are run, but not all platforms can guarantee |
7453a54e SL |
67 | /// that destructors will be run for all types in thread local storage. For |
68 | /// example, there are a number of known caveats where destructors are not run: | |
69 | /// | |
70 | /// 1. On Unix systems when pthread-based TLS is being used, destructors will | |
71 | /// not be run for TLS values on the main thread when it exits. Note that the | |
72 | /// application will exit immediately after the main thread exits as well. | |
73 | /// 2. On all platforms it's possible for TLS to re-initialize other TLS slots | |
74 | /// during destruction. Some platforms ensure that this cannot happen | |
75 | /// infinitely by preventing re-initialization of any slot that has been | |
76 | /// destroyed, but not all platforms have this guard. Those platforms that do | |
77 | /// not guard typically have a synthetic limit after which point no more | |
78 | /// destructors are run. | |
7cac9316 | 79 | /// |
3dfed10e | 80 | /// [`with`]: LocalKey::with |
85aaf69f | 81 | #[stable(feature = "rust1", since = "1.0.0")] |
9cc50fc6 SL |
82 | pub struct LocalKey<T: 'static> { |
83 | // This outer `LocalKey<T>` type is what's going to be stored in statics, | |
84 | // but actual data inside will sometimes be tagged with #[thread_local]. | |
85 | // It's not valid for a true static to reference a #[thread_local] static, | |
86 | // so we get around that by exposing an accessor through a layer of function | |
87 | // indirection (this thunk). | |
1a4d82fc | 88 | // |
9cc50fc6 SL |
89 | // Note that the thunk is itself unsafe because the returned lifetime of the |
90 | // slot where data lives, `'static`, is not actually valid. The lifetime | |
3b2f2976 | 91 | // here is actually slightly shorter than the currently running thread! |
9cc50fc6 SL |
92 | // |
93 | // Although this is an extra layer of indirection, it should in theory be | |
94 | // trivially devirtualizable by LLVM because the value of `inner` never | |
95 | // changes and the constant should be readonly within a crate. This mainly | |
96 | // only runs into problems when TLS statics are exported across crates. | |
dc9dc135 | 97 | inner: unsafe fn() -> Option<&'static T>, |
1a4d82fc JJ |
98 | } |
99 | ||
8bb4bdeb | 100 | #[stable(feature = "std_debug", since = "1.16.0")] |
32a655c1 | 101 | impl<T: 'static> fmt::Debug for LocalKey<T> { |
532ac7d7 | 102 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { |
cdc7bbd5 | 103 | f.debug_struct("LocalKey").finish_non_exhaustive() |
32a655c1 SL |
104 | } |
105 | } | |
106 | ||
7cac9316 | 107 | /// Declare a new thread local storage key of type [`std::thread::LocalKey`]. |
62682a34 | 108 | /// |
3157f602 XL |
109 | /// # Syntax |
110 | /// | |
111 | /// The macro wraps any number of static declarations and makes them thread local. | |
041b39d2 | 112 | /// Publicity and attributes for each static are allowed. Example: |
3157f602 XL |
113 | /// |
114 | /// ``` | |
115 | /// use std::cell::RefCell; | |
116 | /// thread_local! { | |
117 | /// pub static FOO: RefCell<u32> = RefCell::new(1); | |
118 | /// | |
119 | /// #[allow(unused)] | |
120 | /// static BAR: RefCell<f32> = RefCell::new(1.0); | |
121 | /// } | |
122 | /// # fn main() {} | |
123 | /// ``` | |
124 | /// | |
3dfed10e | 125 | /// See [`LocalKey` documentation][`std::thread::LocalKey`] for more |
62682a34 | 126 | /// information. |
7cac9316 | 127 | /// |
3dfed10e | 128 | /// [`std::thread::LocalKey`]: crate::thread::LocalKey |
1a4d82fc | 129 | #[macro_export] |
62682a34 | 130 | #[stable(feature = "rust1", since = "1.0.0")] |
532ac7d7 | 131 | #[allow_internal_unstable(thread_local_internals)] |
041b39d2 XL |
132 | macro_rules! thread_local { |
133 | // empty (base case for the recursion) | |
134 | () => {}; | |
135 | ||
cdc7bbd5 XL |
136 | ($(#[$attr:meta])* $vis:vis static $name:ident: $t:ty = const { $init:expr }; $($rest:tt)*) => ( |
137 | $crate::__thread_local_inner!($(#[$attr])* $vis $name, $t, const $init); | |
138 | $crate::thread_local!($($rest)*); | |
139 | ); | |
140 | ||
141 | ($(#[$attr:meta])* $vis:vis static $name:ident: $t:ty = const { $init:expr }) => ( | |
142 | $crate::__thread_local_inner!($(#[$attr])* $vis $name, $t, const $init); | |
143 | ); | |
144 | ||
041b39d2 XL |
145 | // process multiple declarations |
146 | ($(#[$attr:meta])* $vis:vis static $name:ident: $t:ty = $init:expr; $($rest:tt)*) => ( | |
a1dfa0c6 XL |
147 | $crate::__thread_local_inner!($(#[$attr])* $vis $name, $t, $init); |
148 | $crate::thread_local!($($rest)*); | |
041b39d2 XL |
149 | ); |
150 | ||
151 | // handle a single declaration | |
152 | ($(#[$attr:meta])* $vis:vis static $name:ident: $t:ty = $init:expr) => ( | |
a1dfa0c6 | 153 | $crate::__thread_local_inner!($(#[$attr])* $vis $name, $t, $init); |
041b39d2 XL |
154 | ); |
155 | } | |
156 | ||
041b39d2 | 157 | #[doc(hidden)] |
dfeec247 | 158 | #[unstable(feature = "thread_local_internals", reason = "should not be necessary", issue = "none")] |
041b39d2 | 159 | #[macro_export] |
532ac7d7 | 160 | #[allow_internal_unstable(thread_local_internals, cfg_target_thread_local, thread_local)] |
ea8adc8c | 161 | #[allow_internal_unsafe] |
041b39d2 | 162 | macro_rules! __thread_local_inner { |
cdc7bbd5 XL |
163 | // used to generate the `LocalKey` value for const-initialized thread locals |
164 | (@key $t:ty, const $init:expr) => {{ | |
17df50a5 | 165 | #[cfg_attr(not(windows), inline)] // see comments below |
cdc7bbd5 XL |
166 | unsafe fn __getit() -> $crate::option::Option<&'static $t> { |
167 | const _REQUIRE_UNSTABLE: () = $crate::thread::require_unstable_const_init_thread_local(); | |
3c0e092e | 168 | const INIT_EXPR: $t = $init; |
cdc7bbd5 XL |
169 | |
170 | // wasm without atomics maps directly to `static mut`, and dtors | |
171 | // aren't implemented because thread dtors aren't really a thing | |
172 | // on wasm right now | |
173 | // | |
174 | // FIXME(#84224) this should come after the `target_thread_local` | |
175 | // block. | |
3c0e092e | 176 | #[cfg(all(target_family = "wasm", not(target_feature = "atomics")))] |
cdc7bbd5 | 177 | { |
3c0e092e | 178 | static mut VAL: $t = INIT_EXPR; |
cdc7bbd5 XL |
179 | Some(&VAL) |
180 | } | |
181 | ||
182 | // If the platform has support for `#[thread_local]`, use it. | |
183 | #[cfg(all( | |
184 | target_thread_local, | |
3c0e092e | 185 | not(all(target_family = "wasm", not(target_feature = "atomics"))), |
cdc7bbd5 XL |
186 | ))] |
187 | { | |
3c0e092e XL |
188 | #[thread_local] |
189 | static mut VAL: $t = INIT_EXPR; | |
190 | ||
cdc7bbd5 XL |
191 | // If a dtor isn't needed we can do something "very raw" and |
192 | // just get going. | |
193 | if !$crate::mem::needs_drop::<$t>() { | |
cdc7bbd5 XL |
194 | unsafe { |
195 | return Some(&VAL) | |
196 | } | |
197 | } | |
198 | ||
cdc7bbd5 XL |
199 | // 0 == dtor not registered |
200 | // 1 == dtor registered, dtor not run | |
201 | // 2 == dtor registered and is running or has run | |
202 | #[thread_local] | |
203 | static mut STATE: u8 = 0; | |
204 | ||
205 | unsafe extern "C" fn destroy(ptr: *mut u8) { | |
206 | let ptr = ptr as *mut $t; | |
207 | ||
208 | unsafe { | |
209 | debug_assert_eq!(STATE, 1); | |
210 | STATE = 2; | |
211 | $crate::ptr::drop_in_place(ptr); | |
212 | } | |
213 | } | |
214 | ||
215 | unsafe { | |
216 | match STATE { | |
217 | // 0 == we haven't registered a destructor, so do | |
218 | // so now. | |
219 | 0 => { | |
220 | $crate::thread::__FastLocalKeyInner::<$t>::register_dtor( | |
221 | $crate::ptr::addr_of_mut!(VAL) as *mut u8, | |
222 | destroy, | |
223 | ); | |
224 | STATE = 1; | |
225 | Some(&VAL) | |
226 | } | |
227 | // 1 == the destructor is registered and the value | |
228 | // is valid, so return the pointer. | |
229 | 1 => Some(&VAL), | |
230 | // otherwise the destructor has already run, so we | |
231 | // can't give access. | |
232 | _ => None, | |
233 | } | |
234 | } | |
235 | } | |
236 | ||
237 | // On platforms without `#[thread_local]` we fall back to the | |
238 | // same implementation as below for os thread locals. | |
239 | #[cfg(all( | |
240 | not(target_thread_local), | |
3c0e092e | 241 | not(all(target_family = "wasm", not(target_feature = "atomics"))), |
cdc7bbd5 XL |
242 | ))] |
243 | { | |
244 | #[inline] | |
3c0e092e | 245 | const fn __init() -> $t { INIT_EXPR } |
cdc7bbd5 XL |
246 | static __KEY: $crate::thread::__OsLocalKeyInner<$t> = |
247 | $crate::thread::__OsLocalKeyInner::new(); | |
248 | #[allow(unused_unsafe)] | |
249 | unsafe { __KEY.get(__init) } | |
250 | } | |
251 | } | |
252 | ||
253 | unsafe { | |
254 | $crate::thread::LocalKey::new(__getit) | |
255 | } | |
256 | }}; | |
257 | ||
258 | // used to generate the `LocalKey` value for `thread_local!` | |
60c5eb7d | 259 | (@key $t:ty, $init:expr) => { |
ea8adc8c XL |
260 | { |
261 | #[inline] | |
041b39d2 XL |
262 | fn __init() -> $t { $init } |
263 | ||
17df50a5 XL |
264 | // When reading this function you might ask "why is this inlined |
265 | // everywhere other than Windows?", and that's a very reasonable | |
266 | // question to ask. The short story is that it segfaults rustc if | |
267 | // this function is inlined. The longer story is that Windows looks | |
268 | // to not support `extern` references to thread locals across DLL | |
269 | // boundaries. This appears to at least not be supported in the ABI | |
270 | // that LLVM implements. | |
271 | // | |
272 | // Because of this we never inline on Windows, but we do inline on | |
273 | // other platforms (where external references to thread locals | |
274 | // across DLLs are supported). A better fix for this would be to | |
275 | // inline this function on Windows, but only for "statically linked" | |
276 | // components. For example if two separately compiled rlibs end up | |
277 | // getting linked into a DLL then it's fine to inline this function | |
278 | // across that boundary. It's only not fine to inline this function | |
279 | // across a DLL boundary. Unfortunately rustc doesn't currently | |
280 | // have this sort of logic available in an attribute, and it's not | |
281 | // clear that rustc is even equipped to answer this (it's more of a | |
282 | // Cargo question kinda). This means that, unfortunately, Windows | |
283 | // gets the pessimistic path for now where it's never inlined. | |
284 | // | |
285 | // The issue of "should enable on Windows sometimes" is #84933 | |
286 | #[cfg_attr(not(windows), inline)] | |
dc9dc135 | 287 | unsafe fn __getit() -> $crate::option::Option<&'static $t> { |
3c0e092e | 288 | #[cfg(all(target_family = "wasm", not(target_feature = "atomics")))] |
83c7162d XL |
289 | static __KEY: $crate::thread::__StaticLocalKeyInner<$t> = |
290 | $crate::thread::__StaticLocalKeyInner::new(); | |
291 | ||
041b39d2 | 292 | #[thread_local] |
0bf4aa26 XL |
293 | #[cfg(all( |
294 | target_thread_local, | |
3c0e092e | 295 | not(all(target_family = "wasm", not(target_feature = "atomics"))), |
0bf4aa26 | 296 | ))] |
041b39d2 XL |
297 | static __KEY: $crate::thread::__FastLocalKeyInner<$t> = |
298 | $crate::thread::__FastLocalKeyInner::new(); | |
299 | ||
0bf4aa26 XL |
300 | #[cfg(all( |
301 | not(target_thread_local), | |
3c0e092e | 302 | not(all(target_family = "wasm", not(target_feature = "atomics"))), |
0bf4aa26 | 303 | ))] |
041b39d2 XL |
304 | static __KEY: $crate::thread::__OsLocalKeyInner<$t> = |
305 | $crate::thread::__OsLocalKeyInner::new(); | |
306 | ||
3dfed10e XL |
307 | // FIXME: remove the #[allow(...)] marker when macros don't |
308 | // raise warning for missing/extraneous unsafe blocks anymore. | |
309 | // See https://github.com/rust-lang/rust/issues/74838. | |
310 | #[allow(unused_unsafe)] | |
311 | unsafe { __KEY.get(__init) } | |
041b39d2 XL |
312 | } |
313 | ||
3b2f2976 | 314 | unsafe { |
dc9dc135 | 315 | $crate::thread::LocalKey::new(__getit) |
3b2f2976 | 316 | } |
ea8adc8c XL |
317 | } |
318 | }; | |
cdc7bbd5 | 319 | ($(#[$attr:meta])* $vis:vis $name:ident, $t:ty, $($init:tt)*) => { |
ea8adc8c | 320 | $(#[$attr])* $vis const $name: $crate::thread::LocalKey<$t> = |
cdc7bbd5 | 321 | $crate::__thread_local_inner!(@key $t, $($init)*); |
041b39d2 XL |
322 | } |
323 | } | |
324 | ||
041b39d2 | 325 | /// An error returned by [`LocalKey::try_with`](struct.LocalKey.html#method.try_with). |
0531ce1d | 326 | #[stable(feature = "thread_local_try_with", since = "1.26.0")] |
136023e0 | 327 | #[non_exhaustive] |
416331ca | 328 | #[derive(Clone, Copy, Eq, PartialEq)] |
136023e0 | 329 | pub struct AccessError; |
041b39d2 | 330 | |
0531ce1d | 331 | #[stable(feature = "thread_local_try_with", since = "1.26.0")] |
041b39d2 | 332 | impl fmt::Debug for AccessError { |
532ac7d7 | 333 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { |
041b39d2 XL |
334 | f.debug_struct("AccessError").finish() |
335 | } | |
336 | } | |
337 | ||
0531ce1d | 338 | #[stable(feature = "thread_local_try_with", since = "1.26.0")] |
041b39d2 | 339 | impl fmt::Display for AccessError { |
532ac7d7 | 340 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { |
041b39d2 XL |
341 | fmt::Display::fmt("already destroyed", f) |
342 | } | |
343 | } | |
344 | ||
416331ca XL |
345 | #[stable(feature = "thread_local_try_with", since = "1.26.0")] |
346 | impl Error for AccessError {} | |
347 | ||
c34b1796 | 348 | impl<T: 'static> LocalKey<T> { |
62682a34 | 349 | #[doc(hidden)] |
60c5eb7d XL |
350 | #[unstable( |
351 | feature = "thread_local_internals", | |
352 | reason = "recently added to create a key", | |
dfeec247 | 353 | issue = "none" |
60c5eb7d | 354 | )] |
1b1a35ee | 355 | #[rustc_const_unstable(feature = "thread_local_internals", issue = "none")] |
dc9dc135 | 356 | pub const unsafe fn new(inner: unsafe fn() -> Option<&'static T>) -> LocalKey<T> { |
60c5eb7d | 357 | LocalKey { inner } |
62682a34 SL |
358 | } |
359 | ||
9346a6ac | 360 | /// Acquires a reference to the value in this TLS key. |
1a4d82fc JJ |
361 | /// |
362 | /// This will lazily initialize the value if this thread has not referenced | |
363 | /// this key yet. | |
364 | /// | |
365 | /// # Panics | |
366 | /// | |
367 | /// This function will `panic!()` if the key currently has its | |
368 | /// destructor running, and it **may** panic if the destructor has | |
369 | /// previously been run for this thread. | |
85aaf69f | 370 | #[stable(feature = "rust1", since = "1.0.0")] |
1a4d82fc | 371 | pub fn with<F, R>(&'static self, f: F) -> R |
60c5eb7d XL |
372 | where |
373 | F: FnOnce(&T) -> R, | |
374 | { | |
375 | self.try_with(f).expect( | |
376 | "cannot access a Thread Local Storage value \ | |
377 | during or after destruction", | |
378 | ) | |
1a4d82fc JJ |
379 | } |
380 | ||
041b39d2 XL |
381 | /// Acquires a reference to the value in this TLS key. |
382 | /// | |
383 | /// This will lazily initialize the value if this thread has not referenced | |
384 | /// this key yet. If the key has been destroyed (which may happen if this is called | |
29967ef6 | 385 | /// in a destructor), this function will return an [`AccessError`]. |
041b39d2 XL |
386 | /// |
387 | /// # Panics | |
388 | /// | |
389 | /// This function will still `panic!()` if the key is uninitialized and the | |
390 | /// key's initializer panics. | |
0531ce1d | 391 | #[stable(feature = "thread_local_try_with", since = "1.26.0")] |
ba9703b0 | 392 | #[inline] |
041b39d2 | 393 | pub fn try_with<F, R>(&'static self, f: F) -> Result<R, AccessError> |
0531ce1d XL |
394 | where |
395 | F: FnOnce(&T) -> R, | |
396 | { | |
041b39d2 | 397 | unsafe { |
136023e0 | 398 | let thread_local = (self.inner)().ok_or(AccessError)?; |
dc9dc135 XL |
399 | Ok(f(thread_local)) |
400 | } | |
401 | } | |
402 | } | |
403 | ||
404 | mod lazy { | |
405 | use crate::cell::UnsafeCell; | |
dc9dc135 | 406 | use crate::hint; |
60c5eb7d | 407 | use crate::mem; |
dc9dc135 XL |
408 | |
409 | pub struct LazyKeyInner<T> { | |
410 | inner: UnsafeCell<Option<T>>, | |
411 | } | |
412 | ||
413 | impl<T> LazyKeyInner<T> { | |
414 | pub const fn new() -> LazyKeyInner<T> { | |
60c5eb7d | 415 | LazyKeyInner { inner: UnsafeCell::new(None) } |
dc9dc135 XL |
416 | } |
417 | ||
418 | pub unsafe fn get(&self) -> Option<&'static T> { | |
1b1a35ee XL |
419 | // SAFETY: The caller must ensure no reference is ever handed out to |
420 | // the inner cell nor mutable reference to the Option<T> inside said | |
421 | // cell. This make it safe to hand a reference, though the lifetime | |
422 | // of 'static is itself unsafe, making the get method unsafe. | |
423 | unsafe { (*self.inner.get()).as_ref() } | |
dc9dc135 XL |
424 | } |
425 | ||
1b1a35ee XL |
426 | /// The caller must ensure that no reference is active: this method |
427 | /// needs unique access. | |
dc9dc135 XL |
428 | pub unsafe fn initialize<F: FnOnce() -> T>(&self, init: F) -> &'static T { |
429 | // Execute the initialization up front, *then* move it into our slot, | |
430 | // just in case initialization fails. | |
431 | let value = init(); | |
432 | let ptr = self.inner.get(); | |
433 | ||
1b1a35ee XL |
434 | // SAFETY: |
435 | // | |
dc9dc135 XL |
436 | // note that this can in theory just be `*ptr = Some(value)`, but due to |
437 | // the compiler will currently codegen that pattern with something like: | |
438 | // | |
439 | // ptr::drop_in_place(ptr) | |
440 | // ptr::write(ptr, Some(value)) | |
441 | // | |
442 | // Due to this pattern it's possible for the destructor of the value in | |
443 | // `ptr` (e.g., if this is being recursively initialized) to re-access | |
444 | // TLS, in which case there will be a `&` and `&mut` pointer to the same | |
445 | // value (an aliasing violation). To avoid setting the "I'm running a | |
446 | // destructor" flag we just use `mem::replace` which should sequence the | |
447 | // operations a little differently and make this safe to call. | |
1b1a35ee XL |
448 | // |
449 | // The precondition also ensures that we are the only one accessing | |
450 | // `self` at the moment so replacing is fine. | |
451 | unsafe { | |
452 | let _ = mem::replace(&mut *ptr, Some(value)); | |
453 | } | |
454 | ||
455 | // SAFETY: With the call to `mem::replace` it is guaranteed there is | |
456 | // a `Some` behind `ptr`, not a `None` so `unreachable_unchecked` | |
457 | // will never be reached. | |
458 | unsafe { | |
459 | // After storing `Some` we want to get a reference to the contents of | |
460 | // what we just stored. While we could use `unwrap` here and it should | |
461 | // always work it empirically doesn't seem to always get optimized away, | |
462 | // which means that using something like `try_with` can pull in | |
463 | // panicking code and cause a large size bloat. | |
464 | match *ptr { | |
465 | Some(ref x) => x, | |
466 | None => hint::unreachable_unchecked(), | |
467 | } | |
dc9dc135 XL |
468 | } |
469 | } | |
470 | ||
1b1a35ee XL |
471 | /// The other methods hand out references while taking &self. |
472 | /// As such, callers of this method must ensure no `&` and `&mut` are | |
473 | /// available and used at the same time. | |
dc9dc135 XL |
474 | #[allow(unused)] |
475 | pub unsafe fn take(&mut self) -> Option<T> { | |
1b1a35ee XL |
476 | // SAFETY: See doc comment for this method. |
477 | unsafe { (*self.inner.get()).take() } | |
041b39d2 XL |
478 | } |
479 | } | |
480 | } | |
481 | ||
3c0e092e | 482 | /// On some targets like wasm there's no threads, so no need to generate |
83c7162d XL |
483 | /// thread locals and we can instead just use plain statics! |
484 | #[doc(hidden)] | |
3c0e092e | 485 | #[cfg(all(target_family = "wasm", not(target_feature = "atomics")))] |
83c7162d | 486 | pub mod statik { |
dc9dc135 | 487 | use super::lazy::LazyKeyInner; |
532ac7d7 | 488 | use crate::fmt; |
83c7162d XL |
489 | |
490 | pub struct Key<T> { | |
dc9dc135 | 491 | inner: LazyKeyInner<T>, |
83c7162d XL |
492 | } |
493 | ||
60c5eb7d | 494 | unsafe impl<T> Sync for Key<T> {} |
83c7162d XL |
495 | |
496 | impl<T> fmt::Debug for Key<T> { | |
532ac7d7 | 497 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { |
cdc7bbd5 | 498 | f.debug_struct("Key").finish_non_exhaustive() |
83c7162d XL |
499 | } |
500 | } | |
501 | ||
502 | impl<T> Key<T> { | |
503 | pub const fn new() -> Key<T> { | |
60c5eb7d | 504 | Key { inner: LazyKeyInner::new() } |
83c7162d XL |
505 | } |
506 | ||
dc9dc135 | 507 | pub unsafe fn get(&self, init: fn() -> T) -> Option<&'static T> { |
1b1a35ee XL |
508 | // SAFETY: The caller must ensure no reference is ever handed out to |
509 | // the inner cell nor mutable reference to the Option<T> inside said | |
510 | // cell. This make it safe to hand a reference, though the lifetime | |
511 | // of 'static is itself unsafe, making the get method unsafe. | |
512 | let value = unsafe { | |
513 | match self.inner.get() { | |
514 | Some(ref value) => value, | |
515 | None => self.inner.initialize(init), | |
516 | } | |
dc9dc135 | 517 | }; |
1b1a35ee | 518 | |
dc9dc135 | 519 | Some(value) |
83c7162d XL |
520 | } |
521 | } | |
522 | } | |
523 | ||
041b39d2 XL |
524 | #[doc(hidden)] |
525 | #[cfg(target_thread_local)] | |
526 | pub mod fast { | |
dc9dc135 XL |
527 | use super::lazy::LazyKeyInner; |
528 | use crate::cell::Cell; | |
532ac7d7 XL |
529 | use crate::fmt; |
530 | use crate::mem; | |
3dfed10e | 531 | use crate::sys::thread_local_dtor::register_dtor; |
041b39d2 | 532 | |
dc9dc135 XL |
533 | #[derive(Copy, Clone)] |
534 | enum DtorState { | |
535 | Unregistered, | |
536 | Registered, | |
537 | RunningOrHasRun, | |
538 | } | |
539 | ||
540 | // This data structure has been carefully constructed so that the fast path | |
541 | // only contains one branch on x86. That optimization is necessary to avoid | |
542 | // duplicated tls lookups on OSX. | |
543 | // | |
544 | // LLVM issue: https://bugs.llvm.org/show_bug.cgi?id=41722 | |
041b39d2 | 545 | pub struct Key<T> { |
dc9dc135 XL |
546 | // If `LazyKeyInner::get` returns `None`, that indicates either: |
547 | // * The value has never been initialized | |
548 | // * The value is being recursively initialized | |
549 | // * The value has already been destroyed or is being destroyed | |
550 | // To determine which kind of `None`, check `dtor_state`. | |
551 | // | |
552 | // This is very optimizer friendly for the fast path - initialized but | |
553 | // not yet dropped. | |
554 | inner: LazyKeyInner<T>, | |
041b39d2 XL |
555 | |
556 | // Metadata to keep track of the state of the destructor. Remember that | |
dc9dc135 XL |
557 | // this variable is thread-local, not global. |
558 | dtor_state: Cell<DtorState>, | |
041b39d2 XL |
559 | } |
560 | ||
561 | impl<T> fmt::Debug for Key<T> { | |
532ac7d7 | 562 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { |
cdc7bbd5 | 563 | f.debug_struct("Key").finish_non_exhaustive() |
041b39d2 XL |
564 | } |
565 | } | |
566 | ||
041b39d2 XL |
567 | impl<T> Key<T> { |
568 | pub const fn new() -> Key<T> { | |
60c5eb7d | 569 | Key { inner: LazyKeyInner::new(), dtor_state: Cell::new(DtorState::Unregistered) } |
041b39d2 XL |
570 | } |
571 | ||
cdc7bbd5 XL |
572 | // note that this is just a publically-callable function only for the |
573 | // const-initialized form of thread locals, basically a way to call the | |
574 | // free `register_dtor` function defined elsewhere in libstd. | |
575 | pub unsafe fn register_dtor(a: *mut u8, dtor: unsafe extern "C" fn(*mut u8)) { | |
576 | unsafe { | |
577 | register_dtor(a, dtor); | |
578 | } | |
579 | } | |
580 | ||
dc9dc135 | 581 | pub unsafe fn get<F: FnOnce() -> T>(&self, init: F) -> Option<&'static T> { |
1b1a35ee XL |
582 | // SAFETY: See the definitions of `LazyKeyInner::get` and |
583 | // `try_initialize` for more informations. | |
584 | // | |
585 | // The caller must ensure no mutable references are ever active to | |
586 | // the inner cell or the inner T when this is called. | |
587 | // The `try_initialize` is dependant on the passed `init` function | |
588 | // for this. | |
589 | unsafe { | |
590 | match self.inner.get() { | |
591 | Some(val) => Some(val), | |
592 | None => self.try_initialize(init), | |
593 | } | |
041b39d2 | 594 | } |
041b39d2 XL |
595 | } |
596 | ||
dc9dc135 XL |
597 | // `try_initialize` is only called once per fast thread local variable, |
598 | // except in corner cases where thread_local dtors reference other | |
599 | // thread_local's, or it is being recursively initialized. | |
600 | // | |
601 | // Macos: Inlining this function can cause two `tlv_get_addr` calls to | |
1b1a35ee | 602 | // be performed for every call to `Key::get`. |
dc9dc135 | 603 | // LLVM issue: https://bugs.llvm.org/show_bug.cgi?id=41722 |
1b1a35ee | 604 | #[inline(never)] |
dc9dc135 | 605 | unsafe fn try_initialize<F: FnOnce() -> T>(&self, init: F) -> Option<&'static T> { |
1b1a35ee XL |
606 | // SAFETY: See comment above (this function doc). |
607 | if !mem::needs_drop::<T>() || unsafe { self.try_register_dtor() } { | |
608 | // SAFETY: See comment above (his function doc). | |
609 | Some(unsafe { self.inner.initialize(init) }) | |
dc9dc135 XL |
610 | } else { |
611 | None | |
041b39d2 | 612 | } |
dc9dc135 | 613 | } |
041b39d2 | 614 | |
dc9dc135 XL |
615 | // `try_register_dtor` is only called once per fast thread local |
616 | // variable, except in corner cases where thread_local dtors reference | |
617 | // other thread_local's, or it is being recursively initialized. | |
618 | unsafe fn try_register_dtor(&self) -> bool { | |
619 | match self.dtor_state.get() { | |
620 | DtorState::Unregistered => { | |
1b1a35ee XL |
621 | // SAFETY: dtor registration happens before initialization. |
622 | // Passing `self` as a pointer while using `destroy_value<T>` | |
623 | // is safe because the function will build a pointer to a | |
624 | // Key<T>, which is the type of self and so find the correct | |
625 | // size. | |
626 | unsafe { register_dtor(self as *const _ as *mut u8, destroy_value::<T>) }; | |
dc9dc135 XL |
627 | self.dtor_state.set(DtorState::Registered); |
628 | true | |
629 | } | |
630 | DtorState::Registered => { | |
631 | // recursively initialized | |
632 | true | |
633 | } | |
60c5eb7d | 634 | DtorState::RunningOrHasRun => false, |
dc9dc135 | 635 | } |
041b39d2 XL |
636 | } |
637 | } | |
638 | ||
60c5eb7d | 639 | unsafe extern "C" fn destroy_value<T>(ptr: *mut u8) { |
041b39d2 | 640 | let ptr = ptr as *mut Key<T>; |
041b39d2 | 641 | |
1b1a35ee XL |
642 | // SAFETY: |
643 | // | |
644 | // The pointer `ptr` has been built just above and comes from | |
645 | // `try_register_dtor` where it is originally a Key<T> coming from `self`, | |
646 | // making it non-NUL and of the correct type. | |
647 | // | |
dc9dc135 XL |
648 | // Right before we run the user destructor be sure to set the |
649 | // `Option<T>` to `None`, and `dtor_state` to `RunningOrHasRun`. This | |
650 | // causes future calls to `get` to run `try_initialize_drop` again, | |
651 | // which will now fail, and return `None`. | |
1b1a35ee XL |
652 | unsafe { |
653 | let value = (*ptr).inner.take(); | |
654 | (*ptr).dtor_state.set(DtorState::RunningOrHasRun); | |
655 | drop(value); | |
656 | } | |
041b39d2 | 657 | } |
1a4d82fc JJ |
658 | } |
659 | ||
d9579d0f | 660 | #[doc(hidden)] |
9cc50fc6 | 661 | pub mod os { |
dc9dc135 XL |
662 | use super::lazy::LazyKeyInner; |
663 | use crate::cell::Cell; | |
532ac7d7 XL |
664 | use crate::fmt; |
665 | use crate::marker; | |
666 | use crate::ptr; | |
3dfed10e | 667 | use crate::sys_common::thread_local_key::StaticKey as OsStaticKey; |
1a4d82fc | 668 | |
1a4d82fc | 669 | pub struct Key<T> { |
1a4d82fc | 670 | // OS-TLS key that we'll use to key off. |
62682a34 SL |
671 | os: OsStaticKey, |
672 | marker: marker::PhantomData<Cell<T>>, | |
1a4d82fc JJ |
673 | } |
674 | ||
32a655c1 | 675 | impl<T> fmt::Debug for Key<T> { |
532ac7d7 | 676 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { |
cdc7bbd5 | 677 | f.debug_struct("Key").finish_non_exhaustive() |
32a655c1 SL |
678 | } |
679 | } | |
680 | ||
60c5eb7d | 681 | unsafe impl<T> Sync for Key<T> {} |
1a4d82fc JJ |
682 | |
683 | struct Value<T: 'static> { | |
dc9dc135 | 684 | inner: LazyKeyInner<T>, |
1a4d82fc | 685 | key: &'static Key<T>, |
1a4d82fc JJ |
686 | } |
687 | ||
62682a34 | 688 | impl<T: 'static> Key<T> { |
1b1a35ee | 689 | #[rustc_const_unstable(feature = "thread_local_internals", issue = "none")] |
62682a34 | 690 | pub const fn new() -> Key<T> { |
60c5eb7d | 691 | Key { os: OsStaticKey::new(Some(destroy_value::<T>)), marker: marker::PhantomData } |
1a4d82fc JJ |
692 | } |
693 | ||
1b1a35ee XL |
694 | /// It is a requirement for the caller to ensure that no mutable |
695 | /// reference is active when this method is called. | |
dc9dc135 | 696 | pub unsafe fn get(&'static self, init: fn() -> T) -> Option<&'static T> { |
1b1a35ee XL |
697 | // SAFETY: See the documentation for this method. |
698 | let ptr = unsafe { self.os.get() as *mut Value<T> }; | |
dc9dc135 | 699 | if ptr as usize > 1 { |
1b1a35ee XL |
700 | // SAFETY: the check ensured the pointer is safe (its destructor |
701 | // is not running) + it is coming from a trusted source (self). | |
702 | if let Some(ref value) = unsafe { (*ptr).inner.get() } { | |
e74abb32 | 703 | return Some(value); |
1a4d82fc | 704 | } |
dc9dc135 | 705 | } |
1b1a35ee XL |
706 | // SAFETY: At this point we are sure we have no value and so |
707 | // initializing (or trying to) is safe. | |
708 | unsafe { self.try_initialize(init) } | |
dc9dc135 XL |
709 | } |
710 | ||
711 | // `try_initialize` is only called once per os thread local variable, | |
712 | // except in corner cases where thread_local dtors reference other | |
713 | // thread_local's, or it is being recursively initialized. | |
714 | unsafe fn try_initialize(&'static self, init: fn() -> T) -> Option<&'static T> { | |
1b1a35ee XL |
715 | // SAFETY: No mutable references are ever handed out meaning getting |
716 | // the value is ok. | |
717 | let ptr = unsafe { self.os.get() as *mut Value<T> }; | |
dc9dc135 XL |
718 | if ptr as usize == 1 { |
719 | // destructor is running | |
60c5eb7d | 720 | return None; |
7453a54e | 721 | } |
3b2f2976 | 722 | |
dc9dc135 XL |
723 | let ptr = if ptr.is_null() { |
724 | // If the lookup returned null, we haven't initialized our own | |
725 | // local copy, so do that now. | |
60c5eb7d | 726 | let ptr: Box<Value<T>> = box Value { inner: LazyKeyInner::new(), key: self }; |
dc9dc135 | 727 | let ptr = Box::into_raw(ptr); |
1b1a35ee XL |
728 | // SAFETY: At this point we are sure there is no value inside |
729 | // ptr so setting it will not affect anyone else. | |
730 | unsafe { | |
731 | self.os.set(ptr as *mut u8); | |
732 | } | |
dc9dc135 XL |
733 | ptr |
734 | } else { | |
735 | // recursive initialization | |
736 | ptr | |
3b2f2976 | 737 | }; |
dc9dc135 | 738 | |
1b1a35ee XL |
739 | // SAFETY: ptr has been ensured as non-NUL just above an so can be |
740 | // dereferenced safely. | |
741 | unsafe { Some((*ptr).inner.initialize(init)) } | |
1a4d82fc JJ |
742 | } |
743 | } | |
744 | ||
60c5eb7d | 745 | unsafe extern "C" fn destroy_value<T: 'static>(ptr: *mut u8) { |
1b1a35ee XL |
746 | // SAFETY: |
747 | // | |
17df50a5 | 748 | // The OS TLS ensures that this key contains a null value when this |
1a4d82fc JJ |
749 | // destructor starts to run. We set it back to a sentinel value of 1 to |
750 | // ensure that any future calls to `get` for this thread will return | |
751 | // `None`. | |
752 | // | |
753 | // Note that to prevent an infinite loop we reset it back to null right | |
754 | // before we return from the destructor ourselves. | |
1b1a35ee XL |
755 | unsafe { |
756 | let ptr = Box::from_raw(ptr as *mut Value<T>); | |
757 | let key = ptr.key; | |
758 | key.os.set(1 as *mut u8); | |
759 | drop(ptr); | |
760 | key.os.set(ptr::null_mut()); | |
1a4d82fc | 761 | } |
1a4d82fc JJ |
762 | } |
763 | } |