]>
Commit | Line | Data |
---|---|---|
1a4d82fc JJ |
1 | //===-- tsan_interface_atomic.cc ------------------------------------------===// |
2 | // | |
3 | // The LLVM Compiler Infrastructure | |
4 | // | |
5 | // This file is distributed under the University of Illinois Open Source | |
6 | // License. See LICENSE.TXT for details. | |
7 | // | |
8 | //===----------------------------------------------------------------------===// | |
9 | // | |
10 | // This file is a part of ThreadSanitizer (TSan), a race detector. | |
11 | // | |
12 | //===----------------------------------------------------------------------===// | |
13 | ||
14 | // ThreadSanitizer atomic operations are based on C++11/C1x standards. | |
92a42be0 | 15 | // For background see C++11 standard. A slightly older, publicly |
1a4d82fc JJ |
16 | // available draft of the standard (not entirely up-to-date, but close enough |
17 | // for casual browsing) is available here: | |
18 | // http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2011/n3242.pdf | |
19 | // The following page contains more background information: | |
20 | // http://www.hpl.hp.com/personal/Hans_Boehm/c++mm/ | |
21 | ||
22 | #include "sanitizer_common/sanitizer_placement_new.h" | |
23 | #include "sanitizer_common/sanitizer_stacktrace.h" | |
24 | #include "sanitizer_common/sanitizer_mutex.h" | |
25 | #include "tsan_flags.h" | |
5bcae85e | 26 | #include "tsan_interface.h" |
1a4d82fc JJ |
27 | #include "tsan_rtl.h" |
28 | ||
29 | using namespace __tsan; // NOLINT | |
30 | ||
7cac9316 | 31 | #if !SANITIZER_GO && __TSAN_HAS_INT128 |
1a4d82fc JJ |
32 | // Protects emulation of 128-bit atomic operations. |
33 | static StaticSpinMutex mutex128; | |
92a42be0 | 34 | #endif |
1a4d82fc | 35 | |
1a4d82fc JJ |
36 | static bool IsLoadOrder(morder mo) { |
37 | return mo == mo_relaxed || mo == mo_consume | |
38 | || mo == mo_acquire || mo == mo_seq_cst; | |
39 | } | |
40 | ||
41 | static bool IsStoreOrder(morder mo) { | |
42 | return mo == mo_relaxed || mo == mo_release || mo == mo_seq_cst; | |
43 | } | |
44 | ||
45 | static bool IsReleaseOrder(morder mo) { | |
46 | return mo == mo_release || mo == mo_acq_rel || mo == mo_seq_cst; | |
47 | } | |
48 | ||
49 | static bool IsAcquireOrder(morder mo) { | |
50 | return mo == mo_consume || mo == mo_acquire | |
51 | || mo == mo_acq_rel || mo == mo_seq_cst; | |
52 | } | |
53 | ||
54 | static bool IsAcqRelOrder(morder mo) { | |
55 | return mo == mo_acq_rel || mo == mo_seq_cst; | |
56 | } | |
57 | ||
58 | template<typename T> T func_xchg(volatile T *v, T op) { | |
59 | T res = __sync_lock_test_and_set(v, op); | |
60 | // __sync_lock_test_and_set does not contain full barrier. | |
61 | __sync_synchronize(); | |
62 | return res; | |
63 | } | |
64 | ||
65 | template<typename T> T func_add(volatile T *v, T op) { | |
66 | return __sync_fetch_and_add(v, op); | |
67 | } | |
68 | ||
69 | template<typename T> T func_sub(volatile T *v, T op) { | |
70 | return __sync_fetch_and_sub(v, op); | |
71 | } | |
72 | ||
73 | template<typename T> T func_and(volatile T *v, T op) { | |
74 | return __sync_fetch_and_and(v, op); | |
75 | } | |
76 | ||
77 | template<typename T> T func_or(volatile T *v, T op) { | |
78 | return __sync_fetch_and_or(v, op); | |
79 | } | |
80 | ||
81 | template<typename T> T func_xor(volatile T *v, T op) { | |
82 | return __sync_fetch_and_xor(v, op); | |
83 | } | |
84 | ||
85 | template<typename T> T func_nand(volatile T *v, T op) { | |
86 | // clang does not support __sync_fetch_and_nand. | |
87 | T cmp = *v; | |
88 | for (;;) { | |
89 | T newv = ~(cmp & op); | |
90 | T cur = __sync_val_compare_and_swap(v, cmp, newv); | |
91 | if (cmp == cur) | |
92 | return cmp; | |
93 | cmp = cur; | |
94 | } | |
95 | } | |
96 | ||
97 | template<typename T> T func_cas(volatile T *v, T cmp, T xch) { | |
98 | return __sync_val_compare_and_swap(v, cmp, xch); | |
99 | } | |
100 | ||
101 | // clang does not support 128-bit atomic ops. | |
102 | // Atomic ops are executed under tsan internal mutex, | |
103 | // here we assume that the atomic variables are not accessed | |
104 | // from non-instrumented code. | |
7cac9316 | 105 | #if !defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_16) && !SANITIZER_GO \ |
92a42be0 | 106 | && __TSAN_HAS_INT128 |
1a4d82fc JJ |
107 | a128 func_xchg(volatile a128 *v, a128 op) { |
108 | SpinMutexLock lock(&mutex128); | |
109 | a128 cmp = *v; | |
110 | *v = op; | |
111 | return cmp; | |
112 | } | |
113 | ||
114 | a128 func_add(volatile a128 *v, a128 op) { | |
115 | SpinMutexLock lock(&mutex128); | |
116 | a128 cmp = *v; | |
117 | *v = cmp + op; | |
118 | return cmp; | |
119 | } | |
120 | ||
121 | a128 func_sub(volatile a128 *v, a128 op) { | |
122 | SpinMutexLock lock(&mutex128); | |
123 | a128 cmp = *v; | |
124 | *v = cmp - op; | |
125 | return cmp; | |
126 | } | |
127 | ||
128 | a128 func_and(volatile a128 *v, a128 op) { | |
129 | SpinMutexLock lock(&mutex128); | |
130 | a128 cmp = *v; | |
131 | *v = cmp & op; | |
132 | return cmp; | |
133 | } | |
134 | ||
135 | a128 func_or(volatile a128 *v, a128 op) { | |
136 | SpinMutexLock lock(&mutex128); | |
137 | a128 cmp = *v; | |
138 | *v = cmp | op; | |
139 | return cmp; | |
140 | } | |
141 | ||
142 | a128 func_xor(volatile a128 *v, a128 op) { | |
143 | SpinMutexLock lock(&mutex128); | |
144 | a128 cmp = *v; | |
145 | *v = cmp ^ op; | |
146 | return cmp; | |
147 | } | |
148 | ||
149 | a128 func_nand(volatile a128 *v, a128 op) { | |
150 | SpinMutexLock lock(&mutex128); | |
151 | a128 cmp = *v; | |
152 | *v = ~(cmp & op); | |
153 | return cmp; | |
154 | } | |
155 | ||
156 | a128 func_cas(volatile a128 *v, a128 cmp, a128 xch) { | |
157 | SpinMutexLock lock(&mutex128); | |
158 | a128 cur = *v; | |
159 | if (cur == cmp) | |
160 | *v = xch; | |
161 | return cur; | |
162 | } | |
163 | #endif | |
164 | ||
165 | template<typename T> | |
166 | static int SizeLog() { | |
167 | if (sizeof(T) <= 1) | |
168 | return kSizeLog1; | |
169 | else if (sizeof(T) <= 2) | |
170 | return kSizeLog2; | |
171 | else if (sizeof(T) <= 4) | |
172 | return kSizeLog4; | |
173 | else | |
174 | return kSizeLog8; | |
175 | // For 16-byte atomics we also use 8-byte memory access, | |
176 | // this leads to false negatives only in very obscure cases. | |
177 | } | |
178 | ||
7cac9316 | 179 | #if !SANITIZER_GO |
1a4d82fc | 180 | static atomic_uint8_t *to_atomic(const volatile a8 *a) { |
92a42be0 | 181 | return reinterpret_cast<atomic_uint8_t *>(const_cast<a8 *>(a)); |
1a4d82fc JJ |
182 | } |
183 | ||
184 | static atomic_uint16_t *to_atomic(const volatile a16 *a) { | |
92a42be0 | 185 | return reinterpret_cast<atomic_uint16_t *>(const_cast<a16 *>(a)); |
1a4d82fc | 186 | } |
92a42be0 | 187 | #endif |
1a4d82fc JJ |
188 | |
189 | static atomic_uint32_t *to_atomic(const volatile a32 *a) { | |
92a42be0 | 190 | return reinterpret_cast<atomic_uint32_t *>(const_cast<a32 *>(a)); |
1a4d82fc JJ |
191 | } |
192 | ||
193 | static atomic_uint64_t *to_atomic(const volatile a64 *a) { | |
92a42be0 | 194 | return reinterpret_cast<atomic_uint64_t *>(const_cast<a64 *>(a)); |
1a4d82fc JJ |
195 | } |
196 | ||
197 | static memory_order to_mo(morder mo) { | |
198 | switch (mo) { | |
199 | case mo_relaxed: return memory_order_relaxed; | |
200 | case mo_consume: return memory_order_consume; | |
201 | case mo_acquire: return memory_order_acquire; | |
202 | case mo_release: return memory_order_release; | |
203 | case mo_acq_rel: return memory_order_acq_rel; | |
204 | case mo_seq_cst: return memory_order_seq_cst; | |
205 | } | |
206 | CHECK(0); | |
207 | return memory_order_seq_cst; | |
208 | } | |
209 | ||
210 | template<typename T> | |
211 | static T NoTsanAtomicLoad(const volatile T *a, morder mo) { | |
212 | return atomic_load(to_atomic(a), to_mo(mo)); | |
213 | } | |
214 | ||
7cac9316 | 215 | #if __TSAN_HAS_INT128 && !SANITIZER_GO |
1a4d82fc JJ |
216 | static a128 NoTsanAtomicLoad(const volatile a128 *a, morder mo) { |
217 | SpinMutexLock lock(&mutex128); | |
218 | return *a; | |
219 | } | |
92a42be0 | 220 | #endif |
1a4d82fc JJ |
221 | |
222 | template<typename T> | |
223 | static T AtomicLoad(ThreadState *thr, uptr pc, const volatile T *a, | |
224 | morder mo) { | |
225 | CHECK(IsLoadOrder(mo)); | |
226 | // This fast-path is critical for performance. | |
227 | // Assume the access is atomic. | |
228 | if (!IsAcquireOrder(mo)) { | |
229 | MemoryReadAtomic(thr, pc, (uptr)a, SizeLog<T>()); | |
230 | return NoTsanAtomicLoad(a, mo); | |
231 | } | |
92a42be0 | 232 | SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, (uptr)a, false); |
1a4d82fc JJ |
233 | AcquireImpl(thr, pc, &s->clock); |
234 | T v = NoTsanAtomicLoad(a, mo); | |
235 | s->mtx.ReadUnlock(); | |
236 | MemoryReadAtomic(thr, pc, (uptr)a, SizeLog<T>()); | |
237 | return v; | |
238 | } | |
239 | ||
240 | template<typename T> | |
241 | static void NoTsanAtomicStore(volatile T *a, T v, morder mo) { | |
242 | atomic_store(to_atomic(a), v, to_mo(mo)); | |
243 | } | |
244 | ||
7cac9316 | 245 | #if __TSAN_HAS_INT128 && !SANITIZER_GO |
1a4d82fc JJ |
246 | static void NoTsanAtomicStore(volatile a128 *a, a128 v, morder mo) { |
247 | SpinMutexLock lock(&mutex128); | |
248 | *a = v; | |
249 | } | |
92a42be0 | 250 | #endif |
1a4d82fc JJ |
251 | |
252 | template<typename T> | |
253 | static void AtomicStore(ThreadState *thr, uptr pc, volatile T *a, T v, | |
254 | morder mo) { | |
255 | CHECK(IsStoreOrder(mo)); | |
256 | MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>()); | |
257 | // This fast-path is critical for performance. | |
258 | // Assume the access is atomic. | |
259 | // Strictly saying even relaxed store cuts off release sequence, | |
260 | // so must reset the clock. | |
261 | if (!IsReleaseOrder(mo)) { | |
262 | NoTsanAtomicStore(a, v, mo); | |
263 | return; | |
264 | } | |
265 | __sync_synchronize(); | |
92a42be0 | 266 | SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, (uptr)a, true); |
1a4d82fc JJ |
267 | thr->fast_state.IncrementEpoch(); |
268 | // Can't increment epoch w/o writing to the trace as well. | |
269 | TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0); | |
7cac9316 | 270 | ReleaseStoreImpl(thr, pc, &s->clock); |
1a4d82fc JJ |
271 | NoTsanAtomicStore(a, v, mo); |
272 | s->mtx.Unlock(); | |
273 | } | |
274 | ||
275 | template<typename T, T (*F)(volatile T *v, T op)> | |
276 | static T AtomicRMW(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) { | |
277 | MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>()); | |
278 | SyncVar *s = 0; | |
279 | if (mo != mo_relaxed) { | |
92a42be0 | 280 | s = ctx->metamap.GetOrCreateAndLock(thr, pc, (uptr)a, true); |
1a4d82fc JJ |
281 | thr->fast_state.IncrementEpoch(); |
282 | // Can't increment epoch w/o writing to the trace as well. | |
283 | TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0); | |
284 | if (IsAcqRelOrder(mo)) | |
285 | AcquireReleaseImpl(thr, pc, &s->clock); | |
286 | else if (IsReleaseOrder(mo)) | |
287 | ReleaseImpl(thr, pc, &s->clock); | |
288 | else if (IsAcquireOrder(mo)) | |
289 | AcquireImpl(thr, pc, &s->clock); | |
290 | } | |
291 | v = F(a, v); | |
292 | if (s) | |
293 | s->mtx.Unlock(); | |
294 | return v; | |
295 | } | |
296 | ||
297 | template<typename T> | |
298 | static T NoTsanAtomicExchange(volatile T *a, T v, morder mo) { | |
299 | return func_xchg(a, v); | |
300 | } | |
301 | ||
302 | template<typename T> | |
303 | static T NoTsanAtomicFetchAdd(volatile T *a, T v, morder mo) { | |
304 | return func_add(a, v); | |
305 | } | |
306 | ||
307 | template<typename T> | |
308 | static T NoTsanAtomicFetchSub(volatile T *a, T v, morder mo) { | |
309 | return func_sub(a, v); | |
310 | } | |
311 | ||
312 | template<typename T> | |
313 | static T NoTsanAtomicFetchAnd(volatile T *a, T v, morder mo) { | |
314 | return func_and(a, v); | |
315 | } | |
316 | ||
317 | template<typename T> | |
318 | static T NoTsanAtomicFetchOr(volatile T *a, T v, morder mo) { | |
319 | return func_or(a, v); | |
320 | } | |
321 | ||
322 | template<typename T> | |
323 | static T NoTsanAtomicFetchXor(volatile T *a, T v, morder mo) { | |
324 | return func_xor(a, v); | |
325 | } | |
326 | ||
327 | template<typename T> | |
328 | static T NoTsanAtomicFetchNand(volatile T *a, T v, morder mo) { | |
329 | return func_nand(a, v); | |
330 | } | |
331 | ||
332 | template<typename T> | |
333 | static T AtomicExchange(ThreadState *thr, uptr pc, volatile T *a, T v, | |
334 | morder mo) { | |
335 | return AtomicRMW<T, func_xchg>(thr, pc, a, v, mo); | |
336 | } | |
337 | ||
338 | template<typename T> | |
339 | static T AtomicFetchAdd(ThreadState *thr, uptr pc, volatile T *a, T v, | |
340 | morder mo) { | |
341 | return AtomicRMW<T, func_add>(thr, pc, a, v, mo); | |
342 | } | |
343 | ||
344 | template<typename T> | |
345 | static T AtomicFetchSub(ThreadState *thr, uptr pc, volatile T *a, T v, | |
346 | morder mo) { | |
347 | return AtomicRMW<T, func_sub>(thr, pc, a, v, mo); | |
348 | } | |
349 | ||
350 | template<typename T> | |
351 | static T AtomicFetchAnd(ThreadState *thr, uptr pc, volatile T *a, T v, | |
352 | morder mo) { | |
353 | return AtomicRMW<T, func_and>(thr, pc, a, v, mo); | |
354 | } | |
355 | ||
356 | template<typename T> | |
357 | static T AtomicFetchOr(ThreadState *thr, uptr pc, volatile T *a, T v, | |
358 | morder mo) { | |
359 | return AtomicRMW<T, func_or>(thr, pc, a, v, mo); | |
360 | } | |
361 | ||
362 | template<typename T> | |
363 | static T AtomicFetchXor(ThreadState *thr, uptr pc, volatile T *a, T v, | |
364 | morder mo) { | |
365 | return AtomicRMW<T, func_xor>(thr, pc, a, v, mo); | |
366 | } | |
367 | ||
368 | template<typename T> | |
369 | static T AtomicFetchNand(ThreadState *thr, uptr pc, volatile T *a, T v, | |
370 | morder mo) { | |
371 | return AtomicRMW<T, func_nand>(thr, pc, a, v, mo); | |
372 | } | |
373 | ||
374 | template<typename T> | |
375 | static bool NoTsanAtomicCAS(volatile T *a, T *c, T v, morder mo, morder fmo) { | |
376 | return atomic_compare_exchange_strong(to_atomic(a), c, v, to_mo(mo)); | |
377 | } | |
378 | ||
92a42be0 | 379 | #if __TSAN_HAS_INT128 |
1a4d82fc JJ |
380 | static bool NoTsanAtomicCAS(volatile a128 *a, a128 *c, a128 v, |
381 | morder mo, morder fmo) { | |
382 | a128 old = *c; | |
383 | a128 cur = func_cas(a, old, v); | |
384 | if (cur == old) | |
385 | return true; | |
386 | *c = cur; | |
387 | return false; | |
388 | } | |
92a42be0 | 389 | #endif |
1a4d82fc JJ |
390 | |
391 | template<typename T> | |
92a42be0 SL |
392 | static T NoTsanAtomicCAS(volatile T *a, T c, T v, morder mo, morder fmo) { |
393 | NoTsanAtomicCAS(a, &c, v, mo, fmo); | |
394 | return c; | |
1a4d82fc JJ |
395 | } |
396 | ||
397 | template<typename T> | |
398 | static bool AtomicCAS(ThreadState *thr, uptr pc, | |
399 | volatile T *a, T *c, T v, morder mo, morder fmo) { | |
400 | (void)fmo; // Unused because llvm does not pass it yet. | |
401 | MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>()); | |
402 | SyncVar *s = 0; | |
403 | bool write_lock = mo != mo_acquire && mo != mo_consume; | |
404 | if (mo != mo_relaxed) { | |
92a42be0 | 405 | s = ctx->metamap.GetOrCreateAndLock(thr, pc, (uptr)a, write_lock); |
1a4d82fc JJ |
406 | thr->fast_state.IncrementEpoch(); |
407 | // Can't increment epoch w/o writing to the trace as well. | |
408 | TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0); | |
409 | if (IsAcqRelOrder(mo)) | |
410 | AcquireReleaseImpl(thr, pc, &s->clock); | |
411 | else if (IsReleaseOrder(mo)) | |
412 | ReleaseImpl(thr, pc, &s->clock); | |
413 | else if (IsAcquireOrder(mo)) | |
414 | AcquireImpl(thr, pc, &s->clock); | |
415 | } | |
416 | T cc = *c; | |
417 | T pr = func_cas(a, cc, v); | |
418 | if (s) { | |
419 | if (write_lock) | |
420 | s->mtx.Unlock(); | |
421 | else | |
422 | s->mtx.ReadUnlock(); | |
423 | } | |
424 | if (pr == cc) | |
425 | return true; | |
426 | *c = pr; | |
427 | return false; | |
428 | } | |
429 | ||
430 | template<typename T> | |
431 | static T AtomicCAS(ThreadState *thr, uptr pc, | |
432 | volatile T *a, T c, T v, morder mo, morder fmo) { | |
433 | AtomicCAS(thr, pc, a, &c, v, mo, fmo); | |
434 | return c; | |
435 | } | |
436 | ||
7cac9316 | 437 | #if !SANITIZER_GO |
1a4d82fc JJ |
438 | static void NoTsanAtomicFence(morder mo) { |
439 | __sync_synchronize(); | |
440 | } | |
441 | ||
442 | static void AtomicFence(ThreadState *thr, uptr pc, morder mo) { | |
443 | // FIXME(dvyukov): not implemented. | |
444 | __sync_synchronize(); | |
445 | } | |
92a42be0 SL |
446 | #endif |
447 | ||
448 | // Interface functions follow. | |
7cac9316 | 449 | #if !SANITIZER_GO |
92a42be0 SL |
450 | |
451 | // C/C++ | |
452 | ||
453 | #define SCOPED_ATOMIC(func, ...) \ | |
454 | const uptr callpc = (uptr)__builtin_return_address(0); \ | |
455 | uptr pc = StackTrace::GetCurrentPc(); \ | |
456 | mo = flags()->force_seq_cst_atomics ? (morder)mo_seq_cst : mo; \ | |
457 | ThreadState *const thr = cur_thread(); \ | |
458 | if (thr->ignore_interceptors) \ | |
459 | return NoTsanAtomic##func(__VA_ARGS__); \ | |
460 | AtomicStatInc(thr, sizeof(*a), mo, StatAtomic##func); \ | |
461 | ScopedAtomic sa(thr, callpc, a, mo, __func__); \ | |
462 | return Atomic##func(thr, pc, __VA_ARGS__); \ | |
463 | /**/ | |
464 | ||
465 | class ScopedAtomic { | |
466 | public: | |
467 | ScopedAtomic(ThreadState *thr, uptr pc, const volatile void *a, | |
468 | morder mo, const char *func) | |
469 | : thr_(thr) { | |
470 | FuncEntry(thr_, pc); | |
471 | DPrintf("#%d: %s(%p, %d)\n", thr_->tid, func, a, mo); | |
472 | } | |
473 | ~ScopedAtomic() { | |
474 | ProcessPendingSignals(thr_); | |
475 | FuncExit(thr_); | |
476 | } | |
477 | private: | |
478 | ThreadState *thr_; | |
479 | }; | |
480 | ||
481 | static void AtomicStatInc(ThreadState *thr, uptr size, morder mo, StatType t) { | |
482 | StatInc(thr, StatAtomic); | |
483 | StatInc(thr, t); | |
484 | StatInc(thr, size == 1 ? StatAtomic1 | |
485 | : size == 2 ? StatAtomic2 | |
486 | : size == 4 ? StatAtomic4 | |
487 | : size == 8 ? StatAtomic8 | |
488 | : StatAtomic16); | |
489 | StatInc(thr, mo == mo_relaxed ? StatAtomicRelaxed | |
490 | : mo == mo_consume ? StatAtomicConsume | |
491 | : mo == mo_acquire ? StatAtomicAcquire | |
492 | : mo == mo_release ? StatAtomicRelease | |
493 | : mo == mo_acq_rel ? StatAtomicAcq_Rel | |
494 | : StatAtomicSeq_Cst); | |
495 | } | |
1a4d82fc JJ |
496 | |
497 | extern "C" { | |
498 | SANITIZER_INTERFACE_ATTRIBUTE | |
499 | a8 __tsan_atomic8_load(const volatile a8 *a, morder mo) { | |
500 | SCOPED_ATOMIC(Load, a, mo); | |
501 | } | |
502 | ||
503 | SANITIZER_INTERFACE_ATTRIBUTE | |
504 | a16 __tsan_atomic16_load(const volatile a16 *a, morder mo) { | |
505 | SCOPED_ATOMIC(Load, a, mo); | |
506 | } | |
507 | ||
508 | SANITIZER_INTERFACE_ATTRIBUTE | |
509 | a32 __tsan_atomic32_load(const volatile a32 *a, morder mo) { | |
510 | SCOPED_ATOMIC(Load, a, mo); | |
511 | } | |
512 | ||
513 | SANITIZER_INTERFACE_ATTRIBUTE | |
514 | a64 __tsan_atomic64_load(const volatile a64 *a, morder mo) { | |
515 | SCOPED_ATOMIC(Load, a, mo); | |
516 | } | |
517 | ||
518 | #if __TSAN_HAS_INT128 | |
519 | SANITIZER_INTERFACE_ATTRIBUTE | |
520 | a128 __tsan_atomic128_load(const volatile a128 *a, morder mo) { | |
521 | SCOPED_ATOMIC(Load, a, mo); | |
522 | } | |
523 | #endif | |
524 | ||
525 | SANITIZER_INTERFACE_ATTRIBUTE | |
526 | void __tsan_atomic8_store(volatile a8 *a, a8 v, morder mo) { | |
527 | SCOPED_ATOMIC(Store, a, v, mo); | |
528 | } | |
529 | ||
530 | SANITIZER_INTERFACE_ATTRIBUTE | |
531 | void __tsan_atomic16_store(volatile a16 *a, a16 v, morder mo) { | |
532 | SCOPED_ATOMIC(Store, a, v, mo); | |
533 | } | |
534 | ||
535 | SANITIZER_INTERFACE_ATTRIBUTE | |
536 | void __tsan_atomic32_store(volatile a32 *a, a32 v, morder mo) { | |
537 | SCOPED_ATOMIC(Store, a, v, mo); | |
538 | } | |
539 | ||
540 | SANITIZER_INTERFACE_ATTRIBUTE | |
541 | void __tsan_atomic64_store(volatile a64 *a, a64 v, morder mo) { | |
542 | SCOPED_ATOMIC(Store, a, v, mo); | |
543 | } | |
544 | ||
545 | #if __TSAN_HAS_INT128 | |
546 | SANITIZER_INTERFACE_ATTRIBUTE | |
547 | void __tsan_atomic128_store(volatile a128 *a, a128 v, morder mo) { | |
548 | SCOPED_ATOMIC(Store, a, v, mo); | |
549 | } | |
550 | #endif | |
551 | ||
552 | SANITIZER_INTERFACE_ATTRIBUTE | |
553 | a8 __tsan_atomic8_exchange(volatile a8 *a, a8 v, morder mo) { | |
554 | SCOPED_ATOMIC(Exchange, a, v, mo); | |
555 | } | |
556 | ||
557 | SANITIZER_INTERFACE_ATTRIBUTE | |
558 | a16 __tsan_atomic16_exchange(volatile a16 *a, a16 v, morder mo) { | |
559 | SCOPED_ATOMIC(Exchange, a, v, mo); | |
560 | } | |
561 | ||
562 | SANITIZER_INTERFACE_ATTRIBUTE | |
563 | a32 __tsan_atomic32_exchange(volatile a32 *a, a32 v, morder mo) { | |
564 | SCOPED_ATOMIC(Exchange, a, v, mo); | |
565 | } | |
566 | ||
567 | SANITIZER_INTERFACE_ATTRIBUTE | |
568 | a64 __tsan_atomic64_exchange(volatile a64 *a, a64 v, morder mo) { | |
569 | SCOPED_ATOMIC(Exchange, a, v, mo); | |
570 | } | |
571 | ||
572 | #if __TSAN_HAS_INT128 | |
573 | SANITIZER_INTERFACE_ATTRIBUTE | |
574 | a128 __tsan_atomic128_exchange(volatile a128 *a, a128 v, morder mo) { | |
575 | SCOPED_ATOMIC(Exchange, a, v, mo); | |
576 | } | |
577 | #endif | |
578 | ||
579 | SANITIZER_INTERFACE_ATTRIBUTE | |
580 | a8 __tsan_atomic8_fetch_add(volatile a8 *a, a8 v, morder mo) { | |
581 | SCOPED_ATOMIC(FetchAdd, a, v, mo); | |
582 | } | |
583 | ||
584 | SANITIZER_INTERFACE_ATTRIBUTE | |
585 | a16 __tsan_atomic16_fetch_add(volatile a16 *a, a16 v, morder mo) { | |
586 | SCOPED_ATOMIC(FetchAdd, a, v, mo); | |
587 | } | |
588 | ||
589 | SANITIZER_INTERFACE_ATTRIBUTE | |
590 | a32 __tsan_atomic32_fetch_add(volatile a32 *a, a32 v, morder mo) { | |
591 | SCOPED_ATOMIC(FetchAdd, a, v, mo); | |
592 | } | |
593 | ||
594 | SANITIZER_INTERFACE_ATTRIBUTE | |
595 | a64 __tsan_atomic64_fetch_add(volatile a64 *a, a64 v, morder mo) { | |
596 | SCOPED_ATOMIC(FetchAdd, a, v, mo); | |
597 | } | |
598 | ||
599 | #if __TSAN_HAS_INT128 | |
600 | SANITIZER_INTERFACE_ATTRIBUTE | |
601 | a128 __tsan_atomic128_fetch_add(volatile a128 *a, a128 v, morder mo) { | |
602 | SCOPED_ATOMIC(FetchAdd, a, v, mo); | |
603 | } | |
604 | #endif | |
605 | ||
606 | SANITIZER_INTERFACE_ATTRIBUTE | |
607 | a8 __tsan_atomic8_fetch_sub(volatile a8 *a, a8 v, morder mo) { | |
608 | SCOPED_ATOMIC(FetchSub, a, v, mo); | |
609 | } | |
610 | ||
611 | SANITIZER_INTERFACE_ATTRIBUTE | |
612 | a16 __tsan_atomic16_fetch_sub(volatile a16 *a, a16 v, morder mo) { | |
613 | SCOPED_ATOMIC(FetchSub, a, v, mo); | |
614 | } | |
615 | ||
616 | SANITIZER_INTERFACE_ATTRIBUTE | |
617 | a32 __tsan_atomic32_fetch_sub(volatile a32 *a, a32 v, morder mo) { | |
618 | SCOPED_ATOMIC(FetchSub, a, v, mo); | |
619 | } | |
620 | ||
621 | SANITIZER_INTERFACE_ATTRIBUTE | |
622 | a64 __tsan_atomic64_fetch_sub(volatile a64 *a, a64 v, morder mo) { | |
623 | SCOPED_ATOMIC(FetchSub, a, v, mo); | |
624 | } | |
625 | ||
626 | #if __TSAN_HAS_INT128 | |
627 | SANITIZER_INTERFACE_ATTRIBUTE | |
628 | a128 __tsan_atomic128_fetch_sub(volatile a128 *a, a128 v, morder mo) { | |
629 | SCOPED_ATOMIC(FetchSub, a, v, mo); | |
630 | } | |
631 | #endif | |
632 | ||
633 | SANITIZER_INTERFACE_ATTRIBUTE | |
634 | a8 __tsan_atomic8_fetch_and(volatile a8 *a, a8 v, morder mo) { | |
635 | SCOPED_ATOMIC(FetchAnd, a, v, mo); | |
636 | } | |
637 | ||
638 | SANITIZER_INTERFACE_ATTRIBUTE | |
639 | a16 __tsan_atomic16_fetch_and(volatile a16 *a, a16 v, morder mo) { | |
640 | SCOPED_ATOMIC(FetchAnd, a, v, mo); | |
641 | } | |
642 | ||
643 | SANITIZER_INTERFACE_ATTRIBUTE | |
644 | a32 __tsan_atomic32_fetch_and(volatile a32 *a, a32 v, morder mo) { | |
645 | SCOPED_ATOMIC(FetchAnd, a, v, mo); | |
646 | } | |
647 | ||
648 | SANITIZER_INTERFACE_ATTRIBUTE | |
649 | a64 __tsan_atomic64_fetch_and(volatile a64 *a, a64 v, morder mo) { | |
650 | SCOPED_ATOMIC(FetchAnd, a, v, mo); | |
651 | } | |
652 | ||
653 | #if __TSAN_HAS_INT128 | |
654 | SANITIZER_INTERFACE_ATTRIBUTE | |
655 | a128 __tsan_atomic128_fetch_and(volatile a128 *a, a128 v, morder mo) { | |
656 | SCOPED_ATOMIC(FetchAnd, a, v, mo); | |
657 | } | |
658 | #endif | |
659 | ||
660 | SANITIZER_INTERFACE_ATTRIBUTE | |
661 | a8 __tsan_atomic8_fetch_or(volatile a8 *a, a8 v, morder mo) { | |
662 | SCOPED_ATOMIC(FetchOr, a, v, mo); | |
663 | } | |
664 | ||
665 | SANITIZER_INTERFACE_ATTRIBUTE | |
666 | a16 __tsan_atomic16_fetch_or(volatile a16 *a, a16 v, morder mo) { | |
667 | SCOPED_ATOMIC(FetchOr, a, v, mo); | |
668 | } | |
669 | ||
670 | SANITIZER_INTERFACE_ATTRIBUTE | |
671 | a32 __tsan_atomic32_fetch_or(volatile a32 *a, a32 v, morder mo) { | |
672 | SCOPED_ATOMIC(FetchOr, a, v, mo); | |
673 | } | |
674 | ||
675 | SANITIZER_INTERFACE_ATTRIBUTE | |
676 | a64 __tsan_atomic64_fetch_or(volatile a64 *a, a64 v, morder mo) { | |
677 | SCOPED_ATOMIC(FetchOr, a, v, mo); | |
678 | } | |
679 | ||
680 | #if __TSAN_HAS_INT128 | |
681 | SANITIZER_INTERFACE_ATTRIBUTE | |
682 | a128 __tsan_atomic128_fetch_or(volatile a128 *a, a128 v, morder mo) { | |
683 | SCOPED_ATOMIC(FetchOr, a, v, mo); | |
684 | } | |
685 | #endif | |
686 | ||
687 | SANITIZER_INTERFACE_ATTRIBUTE | |
688 | a8 __tsan_atomic8_fetch_xor(volatile a8 *a, a8 v, morder mo) { | |
689 | SCOPED_ATOMIC(FetchXor, a, v, mo); | |
690 | } | |
691 | ||
692 | SANITIZER_INTERFACE_ATTRIBUTE | |
693 | a16 __tsan_atomic16_fetch_xor(volatile a16 *a, a16 v, morder mo) { | |
694 | SCOPED_ATOMIC(FetchXor, a, v, mo); | |
695 | } | |
696 | ||
697 | SANITIZER_INTERFACE_ATTRIBUTE | |
698 | a32 __tsan_atomic32_fetch_xor(volatile a32 *a, a32 v, morder mo) { | |
699 | SCOPED_ATOMIC(FetchXor, a, v, mo); | |
700 | } | |
701 | ||
702 | SANITIZER_INTERFACE_ATTRIBUTE | |
703 | a64 __tsan_atomic64_fetch_xor(volatile a64 *a, a64 v, morder mo) { | |
704 | SCOPED_ATOMIC(FetchXor, a, v, mo); | |
705 | } | |
706 | ||
707 | #if __TSAN_HAS_INT128 | |
708 | SANITIZER_INTERFACE_ATTRIBUTE | |
709 | a128 __tsan_atomic128_fetch_xor(volatile a128 *a, a128 v, morder mo) { | |
710 | SCOPED_ATOMIC(FetchXor, a, v, mo); | |
711 | } | |
712 | #endif | |
713 | ||
714 | SANITIZER_INTERFACE_ATTRIBUTE | |
715 | a8 __tsan_atomic8_fetch_nand(volatile a8 *a, a8 v, morder mo) { | |
716 | SCOPED_ATOMIC(FetchNand, a, v, mo); | |
717 | } | |
718 | ||
719 | SANITIZER_INTERFACE_ATTRIBUTE | |
720 | a16 __tsan_atomic16_fetch_nand(volatile a16 *a, a16 v, morder mo) { | |
721 | SCOPED_ATOMIC(FetchNand, a, v, mo); | |
722 | } | |
723 | ||
724 | SANITIZER_INTERFACE_ATTRIBUTE | |
725 | a32 __tsan_atomic32_fetch_nand(volatile a32 *a, a32 v, morder mo) { | |
726 | SCOPED_ATOMIC(FetchNand, a, v, mo); | |
727 | } | |
728 | ||
729 | SANITIZER_INTERFACE_ATTRIBUTE | |
730 | a64 __tsan_atomic64_fetch_nand(volatile a64 *a, a64 v, morder mo) { | |
731 | SCOPED_ATOMIC(FetchNand, a, v, mo); | |
732 | } | |
733 | ||
734 | #if __TSAN_HAS_INT128 | |
735 | SANITIZER_INTERFACE_ATTRIBUTE | |
736 | a128 __tsan_atomic128_fetch_nand(volatile a128 *a, a128 v, morder mo) { | |
737 | SCOPED_ATOMIC(FetchNand, a, v, mo); | |
738 | } | |
739 | #endif | |
740 | ||
741 | SANITIZER_INTERFACE_ATTRIBUTE | |
742 | int __tsan_atomic8_compare_exchange_strong(volatile a8 *a, a8 *c, a8 v, | |
743 | morder mo, morder fmo) { | |
744 | SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); | |
745 | } | |
746 | ||
747 | SANITIZER_INTERFACE_ATTRIBUTE | |
748 | int __tsan_atomic16_compare_exchange_strong(volatile a16 *a, a16 *c, a16 v, | |
749 | morder mo, morder fmo) { | |
750 | SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); | |
751 | } | |
752 | ||
753 | SANITIZER_INTERFACE_ATTRIBUTE | |
754 | int __tsan_atomic32_compare_exchange_strong(volatile a32 *a, a32 *c, a32 v, | |
755 | morder mo, morder fmo) { | |
756 | SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); | |
757 | } | |
758 | ||
759 | SANITIZER_INTERFACE_ATTRIBUTE | |
760 | int __tsan_atomic64_compare_exchange_strong(volatile a64 *a, a64 *c, a64 v, | |
761 | morder mo, morder fmo) { | |
762 | SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); | |
763 | } | |
764 | ||
765 | #if __TSAN_HAS_INT128 | |
766 | SANITIZER_INTERFACE_ATTRIBUTE | |
767 | int __tsan_atomic128_compare_exchange_strong(volatile a128 *a, a128 *c, a128 v, | |
768 | morder mo, morder fmo) { | |
769 | SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); | |
770 | } | |
771 | #endif | |
772 | ||
773 | SANITIZER_INTERFACE_ATTRIBUTE | |
774 | int __tsan_atomic8_compare_exchange_weak(volatile a8 *a, a8 *c, a8 v, | |
775 | morder mo, morder fmo) { | |
776 | SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); | |
777 | } | |
778 | ||
779 | SANITIZER_INTERFACE_ATTRIBUTE | |
780 | int __tsan_atomic16_compare_exchange_weak(volatile a16 *a, a16 *c, a16 v, | |
781 | morder mo, morder fmo) { | |
782 | SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); | |
783 | } | |
784 | ||
785 | SANITIZER_INTERFACE_ATTRIBUTE | |
786 | int __tsan_atomic32_compare_exchange_weak(volatile a32 *a, a32 *c, a32 v, | |
787 | morder mo, morder fmo) { | |
788 | SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); | |
789 | } | |
790 | ||
791 | SANITIZER_INTERFACE_ATTRIBUTE | |
792 | int __tsan_atomic64_compare_exchange_weak(volatile a64 *a, a64 *c, a64 v, | |
793 | morder mo, morder fmo) { | |
794 | SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); | |
795 | } | |
796 | ||
797 | #if __TSAN_HAS_INT128 | |
798 | SANITIZER_INTERFACE_ATTRIBUTE | |
799 | int __tsan_atomic128_compare_exchange_weak(volatile a128 *a, a128 *c, a128 v, | |
800 | morder mo, morder fmo) { | |
801 | SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); | |
802 | } | |
803 | #endif | |
804 | ||
805 | SANITIZER_INTERFACE_ATTRIBUTE | |
806 | a8 __tsan_atomic8_compare_exchange_val(volatile a8 *a, a8 c, a8 v, | |
807 | morder mo, morder fmo) { | |
808 | SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); | |
809 | } | |
810 | ||
811 | SANITIZER_INTERFACE_ATTRIBUTE | |
812 | a16 __tsan_atomic16_compare_exchange_val(volatile a16 *a, a16 c, a16 v, | |
813 | morder mo, morder fmo) { | |
814 | SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); | |
815 | } | |
816 | ||
817 | SANITIZER_INTERFACE_ATTRIBUTE | |
818 | a32 __tsan_atomic32_compare_exchange_val(volatile a32 *a, a32 c, a32 v, | |
819 | morder mo, morder fmo) { | |
820 | SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); | |
821 | } | |
822 | ||
823 | SANITIZER_INTERFACE_ATTRIBUTE | |
824 | a64 __tsan_atomic64_compare_exchange_val(volatile a64 *a, a64 c, a64 v, | |
825 | morder mo, morder fmo) { | |
826 | SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); | |
827 | } | |
828 | ||
829 | #if __TSAN_HAS_INT128 | |
830 | SANITIZER_INTERFACE_ATTRIBUTE | |
831 | a128 __tsan_atomic128_compare_exchange_val(volatile a128 *a, a128 c, a128 v, | |
832 | morder mo, morder fmo) { | |
833 | SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); | |
834 | } | |
835 | #endif | |
836 | ||
837 | SANITIZER_INTERFACE_ATTRIBUTE | |
838 | void __tsan_atomic_thread_fence(morder mo) { | |
839 | char* a = 0; | |
840 | SCOPED_ATOMIC(Fence, mo); | |
841 | } | |
842 | ||
843 | SANITIZER_INTERFACE_ATTRIBUTE | |
844 | void __tsan_atomic_signal_fence(morder mo) { | |
845 | } | |
846 | } // extern "C" | |
92a42be0 | 847 | |
7cac9316 | 848 | #else // #if !SANITIZER_GO |
92a42be0 SL |
849 | |
850 | // Go | |
851 | ||
852 | #define ATOMIC(func, ...) \ | |
853 | if (thr->ignore_sync) { \ | |
854 | NoTsanAtomic##func(__VA_ARGS__); \ | |
855 | } else { \ | |
856 | FuncEntry(thr, cpc); \ | |
857 | Atomic##func(thr, pc, __VA_ARGS__); \ | |
858 | FuncExit(thr); \ | |
859 | } \ | |
860 | /**/ | |
861 | ||
862 | #define ATOMIC_RET(func, ret, ...) \ | |
863 | if (thr->ignore_sync) { \ | |
864 | (ret) = NoTsanAtomic##func(__VA_ARGS__); \ | |
865 | } else { \ | |
866 | FuncEntry(thr, cpc); \ | |
867 | (ret) = Atomic##func(thr, pc, __VA_ARGS__); \ | |
868 | FuncExit(thr); \ | |
869 | } \ | |
870 | /**/ | |
871 | ||
872 | extern "C" { | |
873 | SANITIZER_INTERFACE_ATTRIBUTE | |
874 | void __tsan_go_atomic32_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { | |
875 | ATOMIC_RET(Load, *(a32*)(a+8), *(a32**)a, mo_acquire); | |
876 | } | |
877 | ||
878 | SANITIZER_INTERFACE_ATTRIBUTE | |
879 | void __tsan_go_atomic64_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { | |
880 | ATOMIC_RET(Load, *(a64*)(a+8), *(a64**)a, mo_acquire); | |
881 | } | |
882 | ||
883 | SANITIZER_INTERFACE_ATTRIBUTE | |
884 | void __tsan_go_atomic32_store(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { | |
885 | ATOMIC(Store, *(a32**)a, *(a32*)(a+8), mo_release); | |
886 | } | |
887 | ||
888 | SANITIZER_INTERFACE_ATTRIBUTE | |
889 | void __tsan_go_atomic64_store(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { | |
890 | ATOMIC(Store, *(a64**)a, *(a64*)(a+8), mo_release); | |
891 | } | |
892 | ||
893 | SANITIZER_INTERFACE_ATTRIBUTE | |
894 | void __tsan_go_atomic32_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { | |
895 | ATOMIC_RET(FetchAdd, *(a32*)(a+16), *(a32**)a, *(a32*)(a+8), mo_acq_rel); | |
896 | } | |
897 | ||
898 | SANITIZER_INTERFACE_ATTRIBUTE | |
899 | void __tsan_go_atomic64_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { | |
900 | ATOMIC_RET(FetchAdd, *(a64*)(a+16), *(a64**)a, *(a64*)(a+8), mo_acq_rel); | |
901 | } | |
902 | ||
903 | SANITIZER_INTERFACE_ATTRIBUTE | |
904 | void __tsan_go_atomic32_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { | |
905 | ATOMIC_RET(Exchange, *(a32*)(a+16), *(a32**)a, *(a32*)(a+8), mo_acq_rel); | |
906 | } | |
907 | ||
908 | SANITIZER_INTERFACE_ATTRIBUTE | |
909 | void __tsan_go_atomic64_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { | |
910 | ATOMIC_RET(Exchange, *(a64*)(a+16), *(a64**)a, *(a64*)(a+8), mo_acq_rel); | |
911 | } | |
912 | ||
913 | SANITIZER_INTERFACE_ATTRIBUTE | |
914 | void __tsan_go_atomic32_compare_exchange( | |
915 | ThreadState *thr, uptr cpc, uptr pc, u8 *a) { | |
916 | a32 cur = 0; | |
917 | a32 cmp = *(a32*)(a+8); | |
918 | ATOMIC_RET(CAS, cur, *(a32**)a, cmp, *(a32*)(a+12), mo_acq_rel, mo_acquire); | |
919 | *(bool*)(a+16) = (cur == cmp); | |
920 | } | |
921 | ||
922 | SANITIZER_INTERFACE_ATTRIBUTE | |
923 | void __tsan_go_atomic64_compare_exchange( | |
924 | ThreadState *thr, uptr cpc, uptr pc, u8 *a) { | |
925 | a64 cur = 0; | |
926 | a64 cmp = *(a64*)(a+8); | |
927 | ATOMIC_RET(CAS, cur, *(a64**)a, cmp, *(a64*)(a+16), mo_acq_rel, mo_acquire); | |
928 | *(bool*)(a+24) = (cur == cmp); | |
929 | } | |
930 | } // extern "C" | |
7cac9316 | 931 | #endif // #if !SANITIZER_GO |