1 //===-- ThreadSanitizer.cpp - race detector -------------------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file is a part of ThreadSanitizer, a race detector.
12 // The tool is under development, for the details about previous versions see
13 // http://code.google.com/p/data-race-test
15 // The instrumentation phase is quite simple:
16 // - Insert calls to run-time library before every memory access.
17 // - Optimizations may apply to avoid instrumenting some of the accesses.
18 // - Insert calls at function entry/exit.
19 // The rest is handled by the run-time library.
20 //===----------------------------------------------------------------------===//
22 #include "llvm/Transforms/Instrumentation.h"
23 #include "llvm/ADT/SmallSet.h"
24 #include "llvm/ADT/SmallString.h"
25 #include "llvm/ADT/SmallVector.h"
26 #include "llvm/ADT/Statistic.h"
27 #include "llvm/ADT/StringExtras.h"
28 #include "llvm/IR/DataLayout.h"
29 #include "llvm/IR/Function.h"
30 #include "llvm/IR/IRBuilder.h"
31 #include "llvm/IR/IntrinsicInst.h"
32 #include "llvm/IR/Intrinsics.h"
33 #include "llvm/IR/LLVMContext.h"
34 #include "llvm/IR/Metadata.h"
35 #include "llvm/IR/Module.h"
36 #include "llvm/IR/Type.h"
37 #include "llvm/Support/CommandLine.h"
38 #include "llvm/Support/Debug.h"
39 #include "llvm/Support/MathExtras.h"
40 #include "llvm/Support/raw_ostream.h"
41 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
42 #include "llvm/Transforms/Utils/ModuleUtils.h"
46 #define DEBUG_TYPE "tsan"
48 static cl::opt
<bool> ClInstrumentMemoryAccesses(
49 "tsan-instrument-memory-accesses", cl::init(true),
50 cl::desc("Instrument memory accesses"), cl::Hidden
);
51 static cl::opt
<bool> ClInstrumentFuncEntryExit(
52 "tsan-instrument-func-entry-exit", cl::init(true),
53 cl::desc("Instrument function entry and exit"), cl::Hidden
);
54 static cl::opt
<bool> ClInstrumentAtomics(
55 "tsan-instrument-atomics", cl::init(true),
56 cl::desc("Instrument atomics"), cl::Hidden
);
57 static cl::opt
<bool> ClInstrumentMemIntrinsics(
58 "tsan-instrument-memintrinsics", cl::init(true),
59 cl::desc("Instrument memintrinsics (memset/memcpy/memmove)"), cl::Hidden
);
61 STATISTIC(NumInstrumentedReads
, "Number of instrumented reads");
62 STATISTIC(NumInstrumentedWrites
, "Number of instrumented writes");
63 STATISTIC(NumOmittedReadsBeforeWrite
,
64 "Number of reads ignored due to following writes");
65 STATISTIC(NumAccessesWithBadSize
, "Number of accesses with bad size");
66 STATISTIC(NumInstrumentedVtableWrites
, "Number of vtable ptr writes");
67 STATISTIC(NumInstrumentedVtableReads
, "Number of vtable ptr reads");
68 STATISTIC(NumOmittedReadsFromConstantGlobals
,
69 "Number of reads from constant globals");
70 STATISTIC(NumOmittedReadsFromVtable
, "Number of vtable reads");
74 /// ThreadSanitizer: instrument the code in module to find races.
75 struct ThreadSanitizer
: public FunctionPass
{
76 ThreadSanitizer() : FunctionPass(ID
), DL(nullptr) {}
77 const char *getPassName() const override
;
78 bool runOnFunction(Function
&F
) override
;
79 bool doInitialization(Module
&M
) override
;
80 static char ID
; // Pass identification, replacement for typeid.
83 void initializeCallbacks(Module
&M
);
84 bool instrumentLoadOrStore(Instruction
*I
);
85 bool instrumentAtomic(Instruction
*I
);
86 bool instrumentMemIntrinsic(Instruction
*I
);
87 void chooseInstructionsToInstrument(SmallVectorImpl
<Instruction
*> &Local
,
88 SmallVectorImpl
<Instruction
*> &All
);
89 bool addrPointsToConstantData(Value
*Addr
);
90 int getMemoryAccessFuncIndex(Value
*Addr
);
95 // Callbacks to run-time library are computed in doInitialization.
96 Function
*TsanFuncEntry
;
97 Function
*TsanFuncExit
;
98 // Accesses sizes are powers of two: 1, 2, 4, 8, 16.
99 static const size_t kNumberOfAccessSizes
= 5;
100 Function
*TsanRead
[kNumberOfAccessSizes
];
101 Function
*TsanWrite
[kNumberOfAccessSizes
];
102 Function
*TsanAtomicLoad
[kNumberOfAccessSizes
];
103 Function
*TsanAtomicStore
[kNumberOfAccessSizes
];
104 Function
*TsanAtomicRMW
[AtomicRMWInst::LAST_BINOP
+ 1][kNumberOfAccessSizes
];
105 Function
*TsanAtomicCAS
[kNumberOfAccessSizes
];
106 Function
*TsanAtomicThreadFence
;
107 Function
*TsanAtomicSignalFence
;
108 Function
*TsanVptrUpdate
;
109 Function
*TsanVptrLoad
;
110 Function
*MemmoveFn
, *MemcpyFn
, *MemsetFn
;
114 char ThreadSanitizer::ID
= 0;
115 INITIALIZE_PASS(ThreadSanitizer
, "tsan",
116 "ThreadSanitizer: detects data races.",
119 const char *ThreadSanitizer::getPassName() const {
120 return "ThreadSanitizer";
123 FunctionPass
*llvm::createThreadSanitizerPass() {
124 return new ThreadSanitizer();
127 static Function
*checkInterfaceFunction(Constant
*FuncOrBitcast
) {
128 if (Function
*F
= dyn_cast
<Function
>(FuncOrBitcast
))
130 FuncOrBitcast
->dump();
131 report_fatal_error("ThreadSanitizer interface function redefined");
134 void ThreadSanitizer::initializeCallbacks(Module
&M
) {
135 IRBuilder
<> IRB(M
.getContext());
136 // Initialize the callbacks.
137 TsanFuncEntry
= checkInterfaceFunction(M
.getOrInsertFunction(
138 "__tsan_func_entry", IRB
.getVoidTy(), IRB
.getInt8PtrTy(), nullptr));
139 TsanFuncExit
= checkInterfaceFunction(M
.getOrInsertFunction(
140 "__tsan_func_exit", IRB
.getVoidTy(), nullptr));
141 OrdTy
= IRB
.getInt32Ty();
142 for (size_t i
= 0; i
< kNumberOfAccessSizes
; ++i
) {
143 const size_t ByteSize
= 1 << i
;
144 const size_t BitSize
= ByteSize
* 8;
145 SmallString
<32> ReadName("__tsan_read" + itostr(ByteSize
));
146 TsanRead
[i
] = checkInterfaceFunction(M
.getOrInsertFunction(
147 ReadName
, IRB
.getVoidTy(), IRB
.getInt8PtrTy(), nullptr));
149 SmallString
<32> WriteName("__tsan_write" + itostr(ByteSize
));
150 TsanWrite
[i
] = checkInterfaceFunction(M
.getOrInsertFunction(
151 WriteName
, IRB
.getVoidTy(), IRB
.getInt8PtrTy(), nullptr));
153 Type
*Ty
= Type::getIntNTy(M
.getContext(), BitSize
);
154 Type
*PtrTy
= Ty
->getPointerTo();
155 SmallString
<32> AtomicLoadName("__tsan_atomic" + itostr(BitSize
) +
157 TsanAtomicLoad
[i
] = checkInterfaceFunction(M
.getOrInsertFunction(
158 AtomicLoadName
, Ty
, PtrTy
, OrdTy
, nullptr));
160 SmallString
<32> AtomicStoreName("__tsan_atomic" + itostr(BitSize
) +
162 TsanAtomicStore
[i
] = checkInterfaceFunction(M
.getOrInsertFunction(
163 AtomicStoreName
, IRB
.getVoidTy(), PtrTy
, Ty
, OrdTy
,
166 for (int op
= AtomicRMWInst::FIRST_BINOP
;
167 op
<= AtomicRMWInst::LAST_BINOP
; ++op
) {
168 TsanAtomicRMW
[op
][i
] = nullptr;
169 const char *NamePart
= nullptr;
170 if (op
== AtomicRMWInst::Xchg
)
171 NamePart
= "_exchange";
172 else if (op
== AtomicRMWInst::Add
)
173 NamePart
= "_fetch_add";
174 else if (op
== AtomicRMWInst::Sub
)
175 NamePart
= "_fetch_sub";
176 else if (op
== AtomicRMWInst::And
)
177 NamePart
= "_fetch_and";
178 else if (op
== AtomicRMWInst::Or
)
179 NamePart
= "_fetch_or";
180 else if (op
== AtomicRMWInst::Xor
)
181 NamePart
= "_fetch_xor";
182 else if (op
== AtomicRMWInst::Nand
)
183 NamePart
= "_fetch_nand";
186 SmallString
<32> RMWName("__tsan_atomic" + itostr(BitSize
) + NamePart
);
187 TsanAtomicRMW
[op
][i
] = checkInterfaceFunction(M
.getOrInsertFunction(
188 RMWName
, Ty
, PtrTy
, Ty
, OrdTy
, nullptr));
191 SmallString
<32> AtomicCASName("__tsan_atomic" + itostr(BitSize
) +
192 "_compare_exchange_val");
193 TsanAtomicCAS
[i
] = checkInterfaceFunction(M
.getOrInsertFunction(
194 AtomicCASName
, Ty
, PtrTy
, Ty
, Ty
, OrdTy
, OrdTy
, nullptr));
196 TsanVptrUpdate
= checkInterfaceFunction(M
.getOrInsertFunction(
197 "__tsan_vptr_update", IRB
.getVoidTy(), IRB
.getInt8PtrTy(),
198 IRB
.getInt8PtrTy(), nullptr));
199 TsanVptrLoad
= checkInterfaceFunction(M
.getOrInsertFunction(
200 "__tsan_vptr_read", IRB
.getVoidTy(), IRB
.getInt8PtrTy(), nullptr));
201 TsanAtomicThreadFence
= checkInterfaceFunction(M
.getOrInsertFunction(
202 "__tsan_atomic_thread_fence", IRB
.getVoidTy(), OrdTy
, nullptr));
203 TsanAtomicSignalFence
= checkInterfaceFunction(M
.getOrInsertFunction(
204 "__tsan_atomic_signal_fence", IRB
.getVoidTy(), OrdTy
, nullptr));
206 MemmoveFn
= checkInterfaceFunction(M
.getOrInsertFunction(
207 "memmove", IRB
.getInt8PtrTy(), IRB
.getInt8PtrTy(),
208 IRB
.getInt8PtrTy(), IntptrTy
, nullptr));
209 MemcpyFn
= checkInterfaceFunction(M
.getOrInsertFunction(
210 "memcpy", IRB
.getInt8PtrTy(), IRB
.getInt8PtrTy(), IRB
.getInt8PtrTy(),
212 MemsetFn
= checkInterfaceFunction(M
.getOrInsertFunction(
213 "memset", IRB
.getInt8PtrTy(), IRB
.getInt8PtrTy(), IRB
.getInt32Ty(),
217 bool ThreadSanitizer::doInitialization(Module
&M
) {
218 DataLayoutPass
*DLP
= getAnalysisIfAvailable
<DataLayoutPass
>();
220 report_fatal_error("data layout missing");
221 DL
= &DLP
->getDataLayout();
223 // Always insert a call to __tsan_init into the module's CTORs.
224 IRBuilder
<> IRB(M
.getContext());
225 IntptrTy
= IRB
.getIntPtrTy(DL
);
226 Value
*TsanInit
= M
.getOrInsertFunction("__tsan_init",
227 IRB
.getVoidTy(), nullptr);
228 appendToGlobalCtors(M
, cast
<Function
>(TsanInit
), 0);
233 static bool isVtableAccess(Instruction
*I
) {
234 if (MDNode
*Tag
= I
->getMetadata(LLVMContext::MD_tbaa
))
235 return Tag
->isTBAAVtableAccess();
239 bool ThreadSanitizer::addrPointsToConstantData(Value
*Addr
) {
240 // If this is a GEP, just analyze its pointer operand.
241 if (GetElementPtrInst
*GEP
= dyn_cast
<GetElementPtrInst
>(Addr
))
242 Addr
= GEP
->getPointerOperand();
244 if (GlobalVariable
*GV
= dyn_cast
<GlobalVariable
>(Addr
)) {
245 if (GV
->isConstant()) {
246 // Reads from constant globals can not race with any writes.
247 NumOmittedReadsFromConstantGlobals
++;
250 } else if (LoadInst
*L
= dyn_cast
<LoadInst
>(Addr
)) {
251 if (isVtableAccess(L
)) {
252 // Reads from a vtable pointer can not race with any writes.
253 NumOmittedReadsFromVtable
++;
260 // Instrumenting some of the accesses may be proven redundant.
261 // Currently handled:
262 // - read-before-write (within same BB, no calls between)
264 // We do not handle some of the patterns that should not survive
265 // after the classic compiler optimizations.
266 // E.g. two reads from the same temp should be eliminated by CSE,
267 // two writes should be eliminated by DSE, etc.
269 // 'Local' is a vector of insns within the same BB (no calls between).
270 // 'All' is a vector of insns that will be instrumented.
271 void ThreadSanitizer::chooseInstructionsToInstrument(
272 SmallVectorImpl
<Instruction
*> &Local
,
273 SmallVectorImpl
<Instruction
*> &All
) {
274 SmallSet
<Value
*, 8> WriteTargets
;
275 // Iterate from the end.
276 for (SmallVectorImpl
<Instruction
*>::reverse_iterator It
= Local
.rbegin(),
277 E
= Local
.rend(); It
!= E
; ++It
) {
278 Instruction
*I
= *It
;
279 if (StoreInst
*Store
= dyn_cast
<StoreInst
>(I
)) {
280 WriteTargets
.insert(Store
->getPointerOperand());
282 LoadInst
*Load
= cast
<LoadInst
>(I
);
283 Value
*Addr
= Load
->getPointerOperand();
284 if (WriteTargets
.count(Addr
)) {
285 // We will write to this temp, so no reason to analyze the read.
286 NumOmittedReadsBeforeWrite
++;
289 if (addrPointsToConstantData(Addr
)) {
290 // Addr points to some constant data -- it can not race with any writes.
299 static bool isAtomic(Instruction
*I
) {
300 if (LoadInst
*LI
= dyn_cast
<LoadInst
>(I
))
301 return LI
->isAtomic() && LI
->getSynchScope() == CrossThread
;
302 if (StoreInst
*SI
= dyn_cast
<StoreInst
>(I
))
303 return SI
->isAtomic() && SI
->getSynchScope() == CrossThread
;
304 if (isa
<AtomicRMWInst
>(I
))
306 if (isa
<AtomicCmpXchgInst
>(I
))
308 if (isa
<FenceInst
>(I
))
313 bool ThreadSanitizer::runOnFunction(Function
&F
) {
314 if (!DL
) return false;
315 initializeCallbacks(*F
.getParent());
316 SmallVector
<Instruction
*, 8> RetVec
;
317 SmallVector
<Instruction
*, 8> AllLoadsAndStores
;
318 SmallVector
<Instruction
*, 8> LocalLoadsAndStores
;
319 SmallVector
<Instruction
*, 8> AtomicAccesses
;
320 SmallVector
<Instruction
*, 8> MemIntrinCalls
;
322 bool HasCalls
= false;
323 bool SanitizeFunction
= F
.hasFnAttribute(Attribute::SanitizeThread
);
325 // Traverse all instructions, collect loads/stores/returns, check for calls.
327 for (auto &Inst
: BB
) {
329 AtomicAccesses
.push_back(&Inst
);
330 else if (isa
<LoadInst
>(Inst
) || isa
<StoreInst
>(Inst
))
331 LocalLoadsAndStores
.push_back(&Inst
);
332 else if (isa
<ReturnInst
>(Inst
))
333 RetVec
.push_back(&Inst
);
334 else if (isa
<CallInst
>(Inst
) || isa
<InvokeInst
>(Inst
)) {
335 if (isa
<MemIntrinsic
>(Inst
))
336 MemIntrinCalls
.push_back(&Inst
);
338 chooseInstructionsToInstrument(LocalLoadsAndStores
, AllLoadsAndStores
);
341 chooseInstructionsToInstrument(LocalLoadsAndStores
, AllLoadsAndStores
);
344 // We have collected all loads and stores.
345 // FIXME: many of these accesses do not need to be checked for races
346 // (e.g. variables that do not escape, etc).
348 // Instrument memory accesses only if we want to report bugs in the function.
349 if (ClInstrumentMemoryAccesses
&& SanitizeFunction
)
350 for (auto Inst
: AllLoadsAndStores
) {
351 Res
|= instrumentLoadOrStore(Inst
);
354 // Instrument atomic memory accesses in any case (they can be used to
355 // implement synchronization).
356 if (ClInstrumentAtomics
)
357 for (auto Inst
: AtomicAccesses
) {
358 Res
|= instrumentAtomic(Inst
);
361 if (ClInstrumentMemIntrinsics
&& SanitizeFunction
)
362 for (auto Inst
: MemIntrinCalls
) {
363 Res
|= instrumentMemIntrinsic(Inst
);
366 // Instrument function entry/exit points if there were instrumented accesses.
367 if ((Res
|| HasCalls
) && ClInstrumentFuncEntryExit
) {
368 IRBuilder
<> IRB(F
.getEntryBlock().getFirstNonPHI());
369 Value
*ReturnAddress
= IRB
.CreateCall(
370 Intrinsic::getDeclaration(F
.getParent(), Intrinsic::returnaddress
),
372 IRB
.CreateCall(TsanFuncEntry
, ReturnAddress
);
373 for (auto RetInst
: RetVec
) {
374 IRBuilder
<> IRBRet(RetInst
);
375 IRBRet
.CreateCall(TsanFuncExit
);
382 bool ThreadSanitizer::instrumentLoadOrStore(Instruction
*I
) {
384 bool IsWrite
= isa
<StoreInst
>(*I
);
385 Value
*Addr
= IsWrite
386 ? cast
<StoreInst
>(I
)->getPointerOperand()
387 : cast
<LoadInst
>(I
)->getPointerOperand();
388 int Idx
= getMemoryAccessFuncIndex(Addr
);
391 if (IsWrite
&& isVtableAccess(I
)) {
392 DEBUG(dbgs() << " VPTR : " << *I
<< "\n");
393 Value
*StoredValue
= cast
<StoreInst
>(I
)->getValueOperand();
394 // StoredValue may be a vector type if we are storing several vptrs at once.
395 // In this case, just take the first element of the vector since this is
396 // enough to find vptr races.
397 if (isa
<VectorType
>(StoredValue
->getType()))
398 StoredValue
= IRB
.CreateExtractElement(
399 StoredValue
, ConstantInt::get(IRB
.getInt32Ty(), 0));
400 if (StoredValue
->getType()->isIntegerTy())
401 StoredValue
= IRB
.CreateIntToPtr(StoredValue
, IRB
.getInt8PtrTy());
402 // Call TsanVptrUpdate.
403 IRB
.CreateCall2(TsanVptrUpdate
,
404 IRB
.CreatePointerCast(Addr
, IRB
.getInt8PtrTy()),
405 IRB
.CreatePointerCast(StoredValue
, IRB
.getInt8PtrTy()));
406 NumInstrumentedVtableWrites
++;
409 if (!IsWrite
&& isVtableAccess(I
)) {
410 IRB
.CreateCall(TsanVptrLoad
,
411 IRB
.CreatePointerCast(Addr
, IRB
.getInt8PtrTy()));
412 NumInstrumentedVtableReads
++;
415 Value
*OnAccessFunc
= IsWrite
? TsanWrite
[Idx
] : TsanRead
[Idx
];
416 IRB
.CreateCall(OnAccessFunc
, IRB
.CreatePointerCast(Addr
, IRB
.getInt8PtrTy()));
417 if (IsWrite
) NumInstrumentedWrites
++;
418 else NumInstrumentedReads
++;
422 static ConstantInt
*createOrdering(IRBuilder
<> *IRB
, AtomicOrdering ord
) {
425 case NotAtomic
: llvm_unreachable("unexpected atomic ordering!");
426 case Unordered
: // Fall-through.
427 case Monotonic
: v
= 0; break;
428 // case Consume: v = 1; break; // Not specified yet.
429 case Acquire
: v
= 2; break;
430 case Release
: v
= 3; break;
431 case AcquireRelease
: v
= 4; break;
432 case SequentiallyConsistent
: v
= 5; break;
434 return IRB
->getInt32(v
);
437 // If a memset intrinsic gets inlined by the code gen, we will miss races on it.
438 // So, we either need to ensure the intrinsic is not inlined, or instrument it.
439 // We do not instrument memset/memmove/memcpy intrinsics (too complicated),
440 // instead we simply replace them with regular function calls, which are then
441 // intercepted by the run-time.
442 // Since tsan is running after everyone else, the calls should not be
443 // replaced back with intrinsics. If that becomes wrong at some point,
444 // we will need to call e.g. __tsan_memset to avoid the intrinsics.
445 bool ThreadSanitizer::instrumentMemIntrinsic(Instruction
*I
) {
447 if (MemSetInst
*M
= dyn_cast
<MemSetInst
>(I
)) {
448 IRB
.CreateCall3(MemsetFn
,
449 IRB
.CreatePointerCast(M
->getArgOperand(0), IRB
.getInt8PtrTy()),
450 IRB
.CreateIntCast(M
->getArgOperand(1), IRB
.getInt32Ty(), false),
451 IRB
.CreateIntCast(M
->getArgOperand(2), IntptrTy
, false));
452 I
->eraseFromParent();
453 } else if (MemTransferInst
*M
= dyn_cast
<MemTransferInst
>(I
)) {
454 IRB
.CreateCall3(isa
<MemCpyInst
>(M
) ? MemcpyFn
: MemmoveFn
,
455 IRB
.CreatePointerCast(M
->getArgOperand(0), IRB
.getInt8PtrTy()),
456 IRB
.CreatePointerCast(M
->getArgOperand(1), IRB
.getInt8PtrTy()),
457 IRB
.CreateIntCast(M
->getArgOperand(2), IntptrTy
, false));
458 I
->eraseFromParent();
463 // Both llvm and ThreadSanitizer atomic operations are based on C++11/C1x
464 // standards. For background see C++11 standard. A slightly older, publicly
465 // available draft of the standard (not entirely up-to-date, but close enough
466 // for casual browsing) is available here:
467 // http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2011/n3242.pdf
468 // The following page contains more background information:
469 // http://www.hpl.hp.com/personal/Hans_Boehm/c++mm/
471 bool ThreadSanitizer::instrumentAtomic(Instruction
*I
) {
473 if (LoadInst
*LI
= dyn_cast
<LoadInst
>(I
)) {
474 Value
*Addr
= LI
->getPointerOperand();
475 int Idx
= getMemoryAccessFuncIndex(Addr
);
478 const size_t ByteSize
= 1 << Idx
;
479 const size_t BitSize
= ByteSize
* 8;
480 Type
*Ty
= Type::getIntNTy(IRB
.getContext(), BitSize
);
481 Type
*PtrTy
= Ty
->getPointerTo();
482 Value
*Args
[] = {IRB
.CreatePointerCast(Addr
, PtrTy
),
483 createOrdering(&IRB
, LI
->getOrdering())};
484 CallInst
*C
= CallInst::Create(TsanAtomicLoad
[Idx
], Args
);
485 ReplaceInstWithInst(I
, C
);
487 } else if (StoreInst
*SI
= dyn_cast
<StoreInst
>(I
)) {
488 Value
*Addr
= SI
->getPointerOperand();
489 int Idx
= getMemoryAccessFuncIndex(Addr
);
492 const size_t ByteSize
= 1 << Idx
;
493 const size_t BitSize
= ByteSize
* 8;
494 Type
*Ty
= Type::getIntNTy(IRB
.getContext(), BitSize
);
495 Type
*PtrTy
= Ty
->getPointerTo();
496 Value
*Args
[] = {IRB
.CreatePointerCast(Addr
, PtrTy
),
497 IRB
.CreateIntCast(SI
->getValueOperand(), Ty
, false),
498 createOrdering(&IRB
, SI
->getOrdering())};
499 CallInst
*C
= CallInst::Create(TsanAtomicStore
[Idx
], Args
);
500 ReplaceInstWithInst(I
, C
);
501 } else if (AtomicRMWInst
*RMWI
= dyn_cast
<AtomicRMWInst
>(I
)) {
502 Value
*Addr
= RMWI
->getPointerOperand();
503 int Idx
= getMemoryAccessFuncIndex(Addr
);
506 Function
*F
= TsanAtomicRMW
[RMWI
->getOperation()][Idx
];
509 const size_t ByteSize
= 1 << Idx
;
510 const size_t BitSize
= ByteSize
* 8;
511 Type
*Ty
= Type::getIntNTy(IRB
.getContext(), BitSize
);
512 Type
*PtrTy
= Ty
->getPointerTo();
513 Value
*Args
[] = {IRB
.CreatePointerCast(Addr
, PtrTy
),
514 IRB
.CreateIntCast(RMWI
->getValOperand(), Ty
, false),
515 createOrdering(&IRB
, RMWI
->getOrdering())};
516 CallInst
*C
= CallInst::Create(F
, Args
);
517 ReplaceInstWithInst(I
, C
);
518 } else if (AtomicCmpXchgInst
*CASI
= dyn_cast
<AtomicCmpXchgInst
>(I
)) {
519 Value
*Addr
= CASI
->getPointerOperand();
520 int Idx
= getMemoryAccessFuncIndex(Addr
);
523 const size_t ByteSize
= 1 << Idx
;
524 const size_t BitSize
= ByteSize
* 8;
525 Type
*Ty
= Type::getIntNTy(IRB
.getContext(), BitSize
);
526 Type
*PtrTy
= Ty
->getPointerTo();
527 Value
*Args
[] = {IRB
.CreatePointerCast(Addr
, PtrTy
),
528 IRB
.CreateIntCast(CASI
->getCompareOperand(), Ty
, false),
529 IRB
.CreateIntCast(CASI
->getNewValOperand(), Ty
, false),
530 createOrdering(&IRB
, CASI
->getSuccessOrdering()),
531 createOrdering(&IRB
, CASI
->getFailureOrdering())};
532 CallInst
*C
= IRB
.CreateCall(TsanAtomicCAS
[Idx
], Args
);
533 Value
*Success
= IRB
.CreateICmpEQ(C
, CASI
->getCompareOperand());
535 Value
*Res
= IRB
.CreateInsertValue(UndefValue::get(CASI
->getType()), C
, 0);
536 Res
= IRB
.CreateInsertValue(Res
, Success
, 1);
538 I
->replaceAllUsesWith(Res
);
539 I
->eraseFromParent();
540 } else if (FenceInst
*FI
= dyn_cast
<FenceInst
>(I
)) {
541 Value
*Args
[] = {createOrdering(&IRB
, FI
->getOrdering())};
542 Function
*F
= FI
->getSynchScope() == SingleThread
?
543 TsanAtomicSignalFence
: TsanAtomicThreadFence
;
544 CallInst
*C
= CallInst::Create(F
, Args
);
545 ReplaceInstWithInst(I
, C
);
550 int ThreadSanitizer::getMemoryAccessFuncIndex(Value
*Addr
) {
551 Type
*OrigPtrTy
= Addr
->getType();
552 Type
*OrigTy
= cast
<PointerType
>(OrigPtrTy
)->getElementType();
553 assert(OrigTy
->isSized());
554 uint32_t TypeSize
= DL
->getTypeStoreSizeInBits(OrigTy
);
555 if (TypeSize
!= 8 && TypeSize
!= 16 &&
556 TypeSize
!= 32 && TypeSize
!= 64 && TypeSize
!= 128) {
557 NumAccessesWithBadSize
++;
558 // Ignore all unusual sizes.
561 size_t Idx
= countTrailingZeros(TypeSize
/ 8);
562 assert(Idx
< kNumberOfAccessSizes
);