]>
Commit | Line | Data |
---|---|---|
970d7e83 LB |
1 | //===-- MemorySanitizer.cpp - detector of uninitialized reads -------------===// |
2 | // | |
3 | // The LLVM Compiler Infrastructure | |
4 | // | |
5 | // This file is distributed under the University of Illinois Open Source | |
6 | // License. See LICENSE.TXT for details. | |
7 | // | |
8 | //===----------------------------------------------------------------------===// | |
9 | /// \file | |
10 | /// This file is a part of MemorySanitizer, a detector of uninitialized | |
11 | /// reads. | |
12 | /// | |
970d7e83 LB |
13 | /// The algorithm of the tool is similar to Memcheck |
14 | /// (http://goo.gl/QKbem). We associate a few shadow bits with every | |
15 | /// byte of the application memory, poison the shadow of the malloc-ed | |
16 | /// or alloca-ed memory, load the shadow bits on every memory read, | |
17 | /// propagate the shadow bits through some of the arithmetic | |
18 | /// instruction (including MOV), store the shadow bits on every memory | |
19 | /// write, report a bug on some other instructions (e.g. JMP) if the | |
20 | /// associated shadow is poisoned. | |
21 | /// | |
22 | /// But there are differences too. The first and the major one: | |
23 | /// compiler instrumentation instead of binary instrumentation. This | |
24 | /// gives us much better register allocation, possible compiler | |
25 | /// optimizations and a fast start-up. But this brings the major issue | |
26 | /// as well: msan needs to see all program events, including system | |
27 | /// calls and reads/writes in system libraries, so we either need to | |
28 | /// compile *everything* with msan or use a binary translation | |
29 | /// component (e.g. DynamoRIO) to instrument pre-built libraries. | |
30 | /// Another difference from Memcheck is that we use 8 shadow bits per | |
31 | /// byte of application memory and use a direct shadow mapping. This | |
32 | /// greatly simplifies the instrumentation code and avoids races on | |
33 | /// shadow updates (Memcheck is single-threaded so races are not a | |
34 | /// concern there. Memcheck uses 2 shadow bits per byte with a slow | |
35 | /// path storage that uses 8 bits per byte). | |
36 | /// | |
37 | /// The default value of shadow is 0, which means "clean" (not poisoned). | |
38 | /// | |
39 | /// Every module initializer should call __msan_init to ensure that the | |
40 | /// shadow memory is ready. On error, __msan_warning is called. Since | |
41 | /// parameters and return values may be passed via registers, we have a | |
42 | /// specialized thread-local shadow for return values | |
43 | /// (__msan_retval_tls) and parameters (__msan_param_tls). | |
44 | /// | |
45 | /// Origin tracking. | |
46 | /// | |
47 | /// MemorySanitizer can track origins (allocation points) of all uninitialized | |
48 | /// values. This behavior is controlled with a flag (msan-track-origins) and is | |
49 | /// disabled by default. | |
50 | /// | |
51 | /// Origins are 4-byte values created and interpreted by the runtime library. | |
52 | /// They are stored in a second shadow mapping, one 4-byte value for 4 bytes | |
53 | /// of application memory. Propagation of origins is basically a bunch of | |
54 | /// "select" instructions that pick the origin of a dirty argument, if an | |
55 | /// instruction has one. | |
56 | /// | |
57 | /// Every 4 aligned, consecutive bytes of application memory have one origin | |
58 | /// value associated with them. If these bytes contain uninitialized data | |
59 | /// coming from 2 different allocations, the last store wins. Because of this, | |
60 | /// MemorySanitizer reports can show unrelated origins, but this is unlikely in | |
61 | /// practice. | |
62 | /// | |
63 | /// Origins are meaningless for fully initialized values, so MemorySanitizer | |
64 | /// avoids storing origin to memory when a fully initialized value is stored. | |
65 | /// This way it avoids needless overwritting origin of the 4-byte region on | |
66 | /// a short (i.e. 1 byte) clean store, and it is also good for performance. | |
1a4d82fc JJ |
67 | /// |
68 | /// Atomic handling. | |
69 | /// | |
70 | /// Ideally, every atomic store of application value should update the | |
71 | /// corresponding shadow location in an atomic way. Unfortunately, atomic store | |
72 | /// of two disjoint locations can not be done without severe slowdown. | |
73 | /// | |
74 | /// Therefore, we implement an approximation that may err on the safe side. | |
75 | /// In this implementation, every atomically accessed location in the program | |
76 | /// may only change from (partially) uninitialized to fully initialized, but | |
77 | /// not the other way around. We load the shadow _after_ the application load, | |
78 | /// and we store the shadow _before_ the app store. Also, we always store clean | |
79 | /// shadow (if the application store is atomic). This way, if the store-load | |
80 | /// pair constitutes a happens-before arc, shadow store and load are correctly | |
81 | /// ordered such that the load will get either the value that was stored, or | |
82 | /// some later value (which is always clean). | |
83 | /// | |
84 | /// This does not work very well with Compare-And-Swap (CAS) and | |
85 | /// Read-Modify-Write (RMW) operations. To follow the above logic, CAS and RMW | |
86 | /// must store the new shadow before the app operation, and load the shadow | |
87 | /// after the app operation. Computers don't work this way. Current | |
88 | /// implementation ignores the load aspect of CAS/RMW, always returning a clean | |
89 | /// value. It implements the store part as a simple atomic store by storing a | |
90 | /// clean shadow. | |
970d7e83 | 91 | |
1a4d82fc | 92 | //===----------------------------------------------------------------------===// |
970d7e83 LB |
93 | |
94 | #include "llvm/Transforms/Instrumentation.h" | |
95 | #include "llvm/ADT/DepthFirstIterator.h" | |
96 | #include "llvm/ADT/SmallString.h" | |
97 | #include "llvm/ADT/SmallVector.h" | |
1a4d82fc JJ |
98 | #include "llvm/ADT/StringExtras.h" |
99 | #include "llvm/ADT/Triple.h" | |
970d7e83 LB |
100 | #include "llvm/IR/DataLayout.h" |
101 | #include "llvm/IR/Function.h" | |
102 | #include "llvm/IR/IRBuilder.h" | |
103 | #include "llvm/IR/InlineAsm.h" | |
1a4d82fc | 104 | #include "llvm/IR/InstVisitor.h" |
970d7e83 LB |
105 | #include "llvm/IR/IntrinsicInst.h" |
106 | #include "llvm/IR/LLVMContext.h" | |
107 | #include "llvm/IR/MDBuilder.h" | |
108 | #include "llvm/IR/Module.h" | |
109 | #include "llvm/IR/Type.h" | |
1a4d82fc | 110 | #include "llvm/IR/ValueMap.h" |
970d7e83 LB |
111 | #include "llvm/Support/CommandLine.h" |
112 | #include "llvm/Support/Compiler.h" | |
113 | #include "llvm/Support/Debug.h" | |
114 | #include "llvm/Support/raw_ostream.h" | |
115 | #include "llvm/Transforms/Utils/BasicBlockUtils.h" | |
970d7e83 LB |
116 | #include "llvm/Transforms/Utils/Local.h" |
117 | #include "llvm/Transforms/Utils/ModuleUtils.h" | |
118 | ||
119 | using namespace llvm; | |
120 | ||
1a4d82fc JJ |
121 | #define DEBUG_TYPE "msan" |
122 | ||
970d7e83 LB |
123 | static const unsigned kMinOriginAlignment = 4; |
124 | static const unsigned kShadowTLSAlignment = 8; | |
125 | ||
85aaf69f SL |
126 | // These constants must be kept in sync with the ones in msan.h. |
127 | static const unsigned kParamTLSSize = 800; | |
128 | static const unsigned kRetvalTLSSize = 800; | |
129 | ||
1a4d82fc JJ |
130 | // Accesses sizes are powers of two: 1, 2, 4, 8. |
131 | static const size_t kNumberOfAccessSizes = 4; | |
132 | ||
970d7e83 LB |
133 | /// \brief Track origins of uninitialized values. |
134 | /// | |
135 | /// Adds a section to MemorySanitizer report that points to the allocation | |
136 | /// (stack or heap) the uninitialized bits came from originally. | |
1a4d82fc | 137 | static cl::opt<int> ClTrackOrigins("msan-track-origins", |
970d7e83 | 138 | cl::desc("Track origins (allocation sites) of poisoned memory"), |
1a4d82fc | 139 | cl::Hidden, cl::init(0)); |
970d7e83 LB |
140 | static cl::opt<bool> ClKeepGoing("msan-keep-going", |
141 | cl::desc("keep going after reporting a UMR"), | |
142 | cl::Hidden, cl::init(false)); | |
143 | static cl::opt<bool> ClPoisonStack("msan-poison-stack", | |
144 | cl::desc("poison uninitialized stack variables"), | |
145 | cl::Hidden, cl::init(true)); | |
146 | static cl::opt<bool> ClPoisonStackWithCall("msan-poison-stack-with-call", | |
147 | cl::desc("poison uninitialized stack variables with a call"), | |
148 | cl::Hidden, cl::init(false)); | |
149 | static cl::opt<int> ClPoisonStackPattern("msan-poison-stack-pattern", | |
150 | cl::desc("poison uninitialized stack variables with the given patter"), | |
151 | cl::Hidden, cl::init(0xff)); | |
1a4d82fc JJ |
152 | static cl::opt<bool> ClPoisonUndef("msan-poison-undef", |
153 | cl::desc("poison undef temps"), | |
154 | cl::Hidden, cl::init(true)); | |
970d7e83 LB |
155 | |
156 | static cl::opt<bool> ClHandleICmp("msan-handle-icmp", | |
157 | cl::desc("propagate shadow through ICmpEQ and ICmpNE"), | |
158 | cl::Hidden, cl::init(true)); | |
159 | ||
160 | static cl::opt<bool> ClHandleICmpExact("msan-handle-icmp-exact", | |
161 | cl::desc("exact handling of relational integer ICmp"), | |
162 | cl::Hidden, cl::init(false)); | |
163 | ||
970d7e83 LB |
164 | // This flag controls whether we check the shadow of the address |
165 | // operand of load or store. Such bugs are very rare, since load from | |
166 | // a garbage address typically results in SEGV, but still happen | |
167 | // (e.g. only lower bits of address are garbage, or the access happens | |
168 | // early at program startup where malloc-ed memory is more likely to | |
169 | // be zeroed. As of 2012-08-28 this flag adds 20% slowdown. | |
170 | static cl::opt<bool> ClCheckAccessAddress("msan-check-access-address", | |
171 | cl::desc("report accesses through a pointer which has poisoned shadow"), | |
172 | cl::Hidden, cl::init(true)); | |
173 | ||
174 | static cl::opt<bool> ClDumpStrictInstructions("msan-dump-strict-instructions", | |
175 | cl::desc("print out instructions with default strict semantics"), | |
176 | cl::Hidden, cl::init(false)); | |
177 | ||
1a4d82fc JJ |
178 | static cl::opt<int> ClInstrumentationWithCallThreshold( |
179 | "msan-instrumentation-with-call-threshold", | |
180 | cl::desc( | |
181 | "If the function being instrumented requires more than " | |
182 | "this number of checks and origin stores, use callbacks instead of " | |
183 | "inline checks (-1 means never use callbacks)."), | |
184 | cl::Hidden, cl::init(3500)); | |
185 | ||
85aaf69f SL |
186 | // This is an experiment to enable handling of cases where shadow is a non-zero |
187 | // compile-time constant. For some unexplainable reason they were silently | |
188 | // ignored in the instrumentation. | |
189 | static cl::opt<bool> ClCheckConstantShadow("msan-check-constant-shadow", | |
190 | cl::desc("Insert checks for constant shadow values"), | |
191 | cl::Hidden, cl::init(false)); | |
970d7e83 LB |
192 | |
193 | namespace { | |
194 | ||
85aaf69f SL |
195 | // Memory map parameters used in application-to-shadow address calculation. |
196 | // Offset = (Addr & ~AndMask) ^ XorMask | |
197 | // Shadow = ShadowBase + Offset | |
198 | // Origin = OriginBase + Offset | |
199 | struct MemoryMapParams { | |
200 | uint64_t AndMask; | |
201 | uint64_t XorMask; | |
202 | uint64_t ShadowBase; | |
203 | uint64_t OriginBase; | |
204 | }; | |
205 | ||
206 | struct PlatformMemoryMapParams { | |
207 | const MemoryMapParams *bits32; | |
208 | const MemoryMapParams *bits64; | |
209 | }; | |
210 | ||
211 | // i386 Linux | |
212 | static const MemoryMapParams LinuxMemoryMapParams32 = { | |
213 | 0x000080000000, // AndMask | |
214 | 0, // XorMask (not used) | |
215 | 0, // ShadowBase (not used) | |
216 | 0x000040000000, // OriginBase | |
217 | }; | |
218 | ||
219 | // x86_64 Linux | |
220 | static const MemoryMapParams LinuxMemoryMapParams64 = { | |
221 | 0x400000000000, // AndMask | |
222 | 0, // XorMask (not used) | |
223 | 0, // ShadowBase (not used) | |
224 | 0x200000000000, // OriginBase | |
225 | }; | |
226 | ||
227 | // i386 FreeBSD | |
228 | static const MemoryMapParams FreeBSDMemoryMapParams32 = { | |
229 | 0x000180000000, // AndMask | |
230 | 0x000040000000, // XorMask | |
231 | 0x000020000000, // ShadowBase | |
232 | 0x000700000000, // OriginBase | |
233 | }; | |
234 | ||
235 | // x86_64 FreeBSD | |
236 | static const MemoryMapParams FreeBSDMemoryMapParams64 = { | |
237 | 0xc00000000000, // AndMask | |
238 | 0x200000000000, // XorMask | |
239 | 0x100000000000, // ShadowBase | |
240 | 0x380000000000, // OriginBase | |
241 | }; | |
242 | ||
243 | static const PlatformMemoryMapParams LinuxMemoryMapParams = { | |
244 | &LinuxMemoryMapParams32, | |
245 | &LinuxMemoryMapParams64, | |
246 | }; | |
247 | ||
248 | static const PlatformMemoryMapParams FreeBSDMemoryMapParams = { | |
249 | &FreeBSDMemoryMapParams32, | |
250 | &FreeBSDMemoryMapParams64, | |
251 | }; | |
252 | ||
970d7e83 LB |
253 | /// \brief An instrumentation pass implementing detection of uninitialized |
254 | /// reads. | |
255 | /// | |
256 | /// MemorySanitizer: instrument the code in module to find | |
257 | /// uninitialized reads. | |
258 | class MemorySanitizer : public FunctionPass { | |
259 | public: | |
1a4d82fc JJ |
260 | MemorySanitizer(int TrackOrigins = 0) |
261 | : FunctionPass(ID), | |
262 | TrackOrigins(std::max(TrackOrigins, (int)ClTrackOrigins)), | |
263 | DL(nullptr), | |
85aaf69f | 264 | WarningFn(nullptr) {} |
1a4d82fc JJ |
265 | const char *getPassName() const override { return "MemorySanitizer"; } |
266 | bool runOnFunction(Function &F) override; | |
267 | bool doInitialization(Module &M) override; | |
970d7e83 LB |
268 | static char ID; // Pass identification, replacement for typeid. |
269 | ||
270 | private: | |
271 | void initializeCallbacks(Module &M); | |
272 | ||
273 | /// \brief Track origins (allocation points) of uninitialized values. | |
1a4d82fc | 274 | int TrackOrigins; |
970d7e83 | 275 | |
1a4d82fc | 276 | const DataLayout *DL; |
970d7e83 LB |
277 | LLVMContext *C; |
278 | Type *IntptrTy; | |
279 | Type *OriginTy; | |
280 | /// \brief Thread-local shadow storage for function parameters. | |
281 | GlobalVariable *ParamTLS; | |
282 | /// \brief Thread-local origin storage for function parameters. | |
283 | GlobalVariable *ParamOriginTLS; | |
284 | /// \brief Thread-local shadow storage for function return value. | |
285 | GlobalVariable *RetvalTLS; | |
286 | /// \brief Thread-local origin storage for function return value. | |
287 | GlobalVariable *RetvalOriginTLS; | |
288 | /// \brief Thread-local shadow storage for in-register va_arg function | |
289 | /// parameters (x86_64-specific). | |
290 | GlobalVariable *VAArgTLS; | |
291 | /// \brief Thread-local shadow storage for va_arg overflow area | |
292 | /// (x86_64-specific). | |
293 | GlobalVariable *VAArgOverflowSizeTLS; | |
294 | /// \brief Thread-local space used to pass origin value to the UMR reporting | |
295 | /// function. | |
296 | GlobalVariable *OriginTLS; | |
297 | ||
298 | /// \brief The run-time callback to print a warning. | |
299 | Value *WarningFn; | |
1a4d82fc JJ |
300 | // These arrays are indexed by log2(AccessSize). |
301 | Value *MaybeWarningFn[kNumberOfAccessSizes]; | |
302 | Value *MaybeStoreOriginFn[kNumberOfAccessSizes]; | |
303 | ||
970d7e83 LB |
304 | /// \brief Run-time helper that generates a new origin value for a stack |
305 | /// allocation. | |
1a4d82fc | 306 | Value *MsanSetAllocaOrigin4Fn; |
970d7e83 LB |
307 | /// \brief Run-time helper that poisons stack on function entry. |
308 | Value *MsanPoisonStackFn; | |
1a4d82fc JJ |
309 | /// \brief Run-time helper that records a store (or any event) of an |
310 | /// uninitialized value and returns an updated origin id encoding this info. | |
311 | Value *MsanChainOriginFn; | |
970d7e83 LB |
312 | /// \brief MSan runtime replacements for memmove, memcpy and memset. |
313 | Value *MemmoveFn, *MemcpyFn, *MemsetFn; | |
314 | ||
85aaf69f SL |
315 | /// \brief Memory map parameters used in application-to-shadow calculation. |
316 | const MemoryMapParams *MapParams; | |
317 | ||
970d7e83 LB |
318 | MDNode *ColdCallWeights; |
319 | /// \brief Branch weights for origin store. | |
320 | MDNode *OriginStoreWeights; | |
970d7e83 LB |
321 | /// \brief An empty volatile inline asm that prevents callback merge. |
322 | InlineAsm *EmptyAsm; | |
323 | ||
324 | friend struct MemorySanitizerVisitor; | |
325 | friend struct VarArgAMD64Helper; | |
326 | }; | |
327 | } // namespace | |
328 | ||
329 | char MemorySanitizer::ID = 0; | |
330 | INITIALIZE_PASS(MemorySanitizer, "msan", | |
331 | "MemorySanitizer: detects uninitialized reads.", | |
332 | false, false) | |
333 | ||
1a4d82fc JJ |
334 | FunctionPass *llvm::createMemorySanitizerPass(int TrackOrigins) { |
335 | return new MemorySanitizer(TrackOrigins); | |
970d7e83 LB |
336 | } |
337 | ||
338 | /// \brief Create a non-const global initialized with the given string. | |
339 | /// | |
340 | /// Creates a writable global for Str so that we can pass it to the | |
341 | /// run-time lib. Runtime uses first 4 bytes of the string to store the | |
342 | /// frame ID, so the string needs to be mutable. | |
343 | static GlobalVariable *createPrivateNonConstGlobalForString(Module &M, | |
344 | StringRef Str) { | |
345 | Constant *StrConst = ConstantDataArray::getString(M.getContext(), Str); | |
346 | return new GlobalVariable(M, StrConst->getType(), /*isConstant=*/false, | |
347 | GlobalValue::PrivateLinkage, StrConst, ""); | |
348 | } | |
349 | ||
350 | ||
351 | /// \brief Insert extern declaration of runtime-provided functions and globals. | |
352 | void MemorySanitizer::initializeCallbacks(Module &M) { | |
353 | // Only do this once. | |
354 | if (WarningFn) | |
355 | return; | |
356 | ||
357 | IRBuilder<> IRB(*C); | |
358 | // Create the callback. | |
359 | // FIXME: this function should have "Cold" calling conv, | |
360 | // which is not yet implemented. | |
361 | StringRef WarningFnName = ClKeepGoing ? "__msan_warning" | |
362 | : "__msan_warning_noreturn"; | |
85aaf69f | 363 | WarningFn = M.getOrInsertFunction(WarningFnName, IRB.getVoidTy(), nullptr); |
970d7e83 | 364 | |
1a4d82fc JJ |
365 | for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes; |
366 | AccessSizeIndex++) { | |
367 | unsigned AccessSize = 1 << AccessSizeIndex; | |
368 | std::string FunctionName = "__msan_maybe_warning_" + itostr(AccessSize); | |
369 | MaybeWarningFn[AccessSizeIndex] = M.getOrInsertFunction( | |
370 | FunctionName, IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8), | |
85aaf69f | 371 | IRB.getInt32Ty(), nullptr); |
1a4d82fc JJ |
372 | |
373 | FunctionName = "__msan_maybe_store_origin_" + itostr(AccessSize); | |
374 | MaybeStoreOriginFn[AccessSizeIndex] = M.getOrInsertFunction( | |
375 | FunctionName, IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8), | |
85aaf69f | 376 | IRB.getInt8PtrTy(), IRB.getInt32Ty(), nullptr); |
1a4d82fc JJ |
377 | } |
378 | ||
379 | MsanSetAllocaOrigin4Fn = M.getOrInsertFunction( | |
380 | "__msan_set_alloca_origin4", IRB.getVoidTy(), IRB.getInt8PtrTy(), IntptrTy, | |
85aaf69f SL |
381 | IRB.getInt8PtrTy(), IntptrTy, nullptr); |
382 | MsanPoisonStackFn = | |
383 | M.getOrInsertFunction("__msan_poison_stack", IRB.getVoidTy(), | |
384 | IRB.getInt8PtrTy(), IntptrTy, nullptr); | |
1a4d82fc | 385 | MsanChainOriginFn = M.getOrInsertFunction( |
85aaf69f | 386 | "__msan_chain_origin", IRB.getInt32Ty(), IRB.getInt32Ty(), nullptr); |
970d7e83 LB |
387 | MemmoveFn = M.getOrInsertFunction( |
388 | "__msan_memmove", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), | |
85aaf69f | 389 | IRB.getInt8PtrTy(), IntptrTy, nullptr); |
970d7e83 LB |
390 | MemcpyFn = M.getOrInsertFunction( |
391 | "__msan_memcpy", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), | |
85aaf69f | 392 | IntptrTy, nullptr); |
970d7e83 LB |
393 | MemsetFn = M.getOrInsertFunction( |
394 | "__msan_memset", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IRB.getInt32Ty(), | |
85aaf69f | 395 | IntptrTy, nullptr); |
970d7e83 LB |
396 | |
397 | // Create globals. | |
398 | RetvalTLS = new GlobalVariable( | |
85aaf69f | 399 | M, ArrayType::get(IRB.getInt64Ty(), kRetvalTLSSize / 8), false, |
1a4d82fc JJ |
400 | GlobalVariable::ExternalLinkage, nullptr, "__msan_retval_tls", nullptr, |
401 | GlobalVariable::InitialExecTLSModel); | |
970d7e83 | 402 | RetvalOriginTLS = new GlobalVariable( |
1a4d82fc JJ |
403 | M, OriginTy, false, GlobalVariable::ExternalLinkage, nullptr, |
404 | "__msan_retval_origin_tls", nullptr, GlobalVariable::InitialExecTLSModel); | |
970d7e83 LB |
405 | |
406 | ParamTLS = new GlobalVariable( | |
85aaf69f | 407 | M, ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), false, |
1a4d82fc JJ |
408 | GlobalVariable::ExternalLinkage, nullptr, "__msan_param_tls", nullptr, |
409 | GlobalVariable::InitialExecTLSModel); | |
970d7e83 | 410 | ParamOriginTLS = new GlobalVariable( |
85aaf69f SL |
411 | M, ArrayType::get(OriginTy, kParamTLSSize / 4), false, |
412 | GlobalVariable::ExternalLinkage, nullptr, "__msan_param_origin_tls", | |
413 | nullptr, GlobalVariable::InitialExecTLSModel); | |
970d7e83 LB |
414 | |
415 | VAArgTLS = new GlobalVariable( | |
85aaf69f | 416 | M, ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), false, |
1a4d82fc JJ |
417 | GlobalVariable::ExternalLinkage, nullptr, "__msan_va_arg_tls", nullptr, |
418 | GlobalVariable::InitialExecTLSModel); | |
970d7e83 | 419 | VAArgOverflowSizeTLS = new GlobalVariable( |
1a4d82fc JJ |
420 | M, IRB.getInt64Ty(), false, GlobalVariable::ExternalLinkage, nullptr, |
421 | "__msan_va_arg_overflow_size_tls", nullptr, | |
422 | GlobalVariable::InitialExecTLSModel); | |
970d7e83 | 423 | OriginTLS = new GlobalVariable( |
1a4d82fc JJ |
424 | M, IRB.getInt32Ty(), false, GlobalVariable::ExternalLinkage, nullptr, |
425 | "__msan_origin_tls", nullptr, GlobalVariable::InitialExecTLSModel); | |
970d7e83 LB |
426 | |
427 | // We insert an empty inline asm after __msan_report* to avoid callback merge. | |
428 | EmptyAsm = InlineAsm::get(FunctionType::get(IRB.getVoidTy(), false), | |
429 | StringRef(""), StringRef(""), | |
430 | /*hasSideEffects=*/true); | |
431 | } | |
432 | ||
433 | /// \brief Module-level initialization. | |
434 | /// | |
435 | /// inserts a call to __msan_init to the module's constructor list. | |
436 | bool MemorySanitizer::doInitialization(Module &M) { | |
1a4d82fc JJ |
437 | DataLayoutPass *DLP = getAnalysisIfAvailable<DataLayoutPass>(); |
438 | if (!DLP) | |
439 | report_fatal_error("data layout missing"); | |
440 | DL = &DLP->getDataLayout(); | |
441 | ||
85aaf69f SL |
442 | Triple TargetTriple(M.getTargetTriple()); |
443 | const PlatformMemoryMapParams *PlatformMapParams; | |
444 | if (TargetTriple.getOS() == Triple::FreeBSD) | |
445 | PlatformMapParams = &FreeBSDMemoryMapParams; | |
446 | else | |
447 | PlatformMapParams = &LinuxMemoryMapParams; | |
448 | ||
970d7e83 | 449 | C = &(M.getContext()); |
1a4d82fc | 450 | unsigned PtrSize = DL->getPointerSizeInBits(/* AddressSpace */0); |
970d7e83 LB |
451 | switch (PtrSize) { |
452 | case 64: | |
85aaf69f | 453 | MapParams = PlatformMapParams->bits64; |
970d7e83 LB |
454 | break; |
455 | case 32: | |
85aaf69f | 456 | MapParams = PlatformMapParams->bits32; |
970d7e83 LB |
457 | break; |
458 | default: | |
459 | report_fatal_error("unsupported pointer size"); | |
460 | break; | |
461 | } | |
462 | ||
463 | IRBuilder<> IRB(*C); | |
1a4d82fc | 464 | IntptrTy = IRB.getIntPtrTy(DL); |
970d7e83 LB |
465 | OriginTy = IRB.getInt32Ty(); |
466 | ||
467 | ColdCallWeights = MDBuilder(*C).createBranchWeights(1, 1000); | |
468 | OriginStoreWeights = MDBuilder(*C).createBranchWeights(1, 1000); | |
469 | ||
470 | // Insert a call to __msan_init/__msan_track_origins into the module's CTORs. | |
471 | appendToGlobalCtors(M, cast<Function>(M.getOrInsertFunction( | |
85aaf69f | 472 | "__msan_init", IRB.getVoidTy(), nullptr)), 0); |
970d7e83 | 473 | |
1a4d82fc JJ |
474 | if (TrackOrigins) |
475 | new GlobalVariable(M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage, | |
476 | IRB.getInt32(TrackOrigins), "__msan_track_origins"); | |
970d7e83 | 477 | |
1a4d82fc JJ |
478 | if (ClKeepGoing) |
479 | new GlobalVariable(M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage, | |
480 | IRB.getInt32(ClKeepGoing), "__msan_keep_going"); | |
970d7e83 LB |
481 | |
482 | return true; | |
483 | } | |
484 | ||
485 | namespace { | |
486 | ||
487 | /// \brief A helper class that handles instrumentation of VarArg | |
488 | /// functions on a particular platform. | |
489 | /// | |
490 | /// Implementations are expected to insert the instrumentation | |
491 | /// necessary to propagate argument shadow through VarArg function | |
492 | /// calls. Visit* methods are called during an InstVisitor pass over | |
493 | /// the function, and should avoid creating new basic blocks. A new | |
494 | /// instance of this class is created for each instrumented function. | |
495 | struct VarArgHelper { | |
496 | /// \brief Visit a CallSite. | |
497 | virtual void visitCallSite(CallSite &CS, IRBuilder<> &IRB) = 0; | |
498 | ||
499 | /// \brief Visit a va_start call. | |
500 | virtual void visitVAStartInst(VAStartInst &I) = 0; | |
501 | ||
502 | /// \brief Visit a va_copy call. | |
503 | virtual void visitVACopyInst(VACopyInst &I) = 0; | |
504 | ||
505 | /// \brief Finalize function instrumentation. | |
506 | /// | |
507 | /// This method is called after visiting all interesting (see above) | |
508 | /// instructions in a function. | |
509 | virtual void finalizeInstrumentation() = 0; | |
510 | ||
511 | virtual ~VarArgHelper() {} | |
512 | }; | |
513 | ||
514 | struct MemorySanitizerVisitor; | |
515 | ||
516 | VarArgHelper* | |
517 | CreateVarArgHelper(Function &Func, MemorySanitizer &Msan, | |
518 | MemorySanitizerVisitor &Visitor); | |
519 | ||
1a4d82fc JJ |
520 | unsigned TypeSizeToSizeIndex(unsigned TypeSize) { |
521 | if (TypeSize <= 8) return 0; | |
522 | return Log2_32_Ceil(TypeSize / 8); | |
523 | } | |
524 | ||
970d7e83 LB |
525 | /// This class does all the work for a given function. Store and Load |
526 | /// instructions store and load corresponding shadow and origin | |
527 | /// values. Most instructions propagate shadow from arguments to their | |
528 | /// return values. Certain instructions (most importantly, BranchInst) | |
529 | /// test their argument shadow and print reports (with a runtime call) if it's | |
530 | /// non-zero. | |
531 | struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { | |
532 | Function &F; | |
533 | MemorySanitizer &MS; | |
534 | SmallVector<PHINode *, 16> ShadowPHINodes, OriginPHINodes; | |
535 | ValueMap<Value*, Value*> ShadowMap, OriginMap; | |
1a4d82fc JJ |
536 | std::unique_ptr<VarArgHelper> VAHelper; |
537 | ||
538 | // The following flags disable parts of MSan instrumentation based on | |
539 | // blacklist contents and command-line options. | |
970d7e83 | 540 | bool InsertChecks; |
1a4d82fc JJ |
541 | bool PropagateShadow; |
542 | bool PoisonStack; | |
543 | bool PoisonUndef; | |
544 | bool CheckReturnValue; | |
970d7e83 LB |
545 | |
546 | struct ShadowOriginAndInsertPoint { | |
1a4d82fc JJ |
547 | Value *Shadow; |
548 | Value *Origin; | |
970d7e83 | 549 | Instruction *OrigIns; |
1a4d82fc | 550 | ShadowOriginAndInsertPoint(Value *S, Value *O, Instruction *I) |
970d7e83 | 551 | : Shadow(S), Origin(O), OrigIns(I) { } |
970d7e83 LB |
552 | }; |
553 | SmallVector<ShadowOriginAndInsertPoint, 16> InstrumentationList; | |
554 | SmallVector<Instruction*, 16> StoreList; | |
555 | ||
556 | MemorySanitizerVisitor(Function &F, MemorySanitizer &MS) | |
557 | : F(F), MS(MS), VAHelper(CreateVarArgHelper(F, MS, *this)) { | |
1a4d82fc JJ |
558 | bool SanitizeFunction = F.getAttributes().hasAttribute( |
559 | AttributeSet::FunctionIndex, Attribute::SanitizeMemory); | |
560 | InsertChecks = SanitizeFunction; | |
561 | PropagateShadow = SanitizeFunction; | |
562 | PoisonStack = SanitizeFunction && ClPoisonStack; | |
563 | PoisonUndef = SanitizeFunction && ClPoisonUndef; | |
564 | // FIXME: Consider using SpecialCaseList to specify a list of functions that | |
565 | // must always return fully initialized values. For now, we hardcode "main". | |
566 | CheckReturnValue = SanitizeFunction && (F.getName() == "main"); | |
970d7e83 LB |
567 | |
568 | DEBUG(if (!InsertChecks) | |
569 | dbgs() << "MemorySanitizer is not inserting checks into '" | |
570 | << F.getName() << "'\n"); | |
571 | } | |
572 | ||
1a4d82fc JJ |
573 | Value *updateOrigin(Value *V, IRBuilder<> &IRB) { |
574 | if (MS.TrackOrigins <= 1) return V; | |
575 | return IRB.CreateCall(MS.MsanChainOriginFn, V); | |
576 | } | |
970d7e83 | 577 | |
1a4d82fc JJ |
578 | void storeOrigin(IRBuilder<> &IRB, Value *Addr, Value *Shadow, Value *Origin, |
579 | unsigned Alignment, bool AsCall) { | |
85aaf69f | 580 | unsigned OriginAlignment = std::max(kMinOriginAlignment, Alignment); |
1a4d82fc | 581 | if (isa<StructType>(Shadow->getType())) { |
85aaf69f SL |
582 | IRB.CreateAlignedStore(updateOrigin(Origin, IRB), |
583 | getOriginPtr(Addr, IRB, Alignment), | |
584 | OriginAlignment); | |
1a4d82fc JJ |
585 | } else { |
586 | Value *ConvertedShadow = convertToShadowTyNoVec(Shadow, IRB); | |
587 | // TODO(eugenis): handle non-zero constant shadow by inserting an | |
588 | // unconditional check (can not simply fail compilation as this could | |
589 | // be in the dead code). | |
85aaf69f SL |
590 | if (!ClCheckConstantShadow) |
591 | if (isa<Constant>(ConvertedShadow)) return; | |
1a4d82fc JJ |
592 | unsigned TypeSizeInBits = |
593 | MS.DL->getTypeSizeInBits(ConvertedShadow->getType()); | |
594 | unsigned SizeIndex = TypeSizeToSizeIndex(TypeSizeInBits); | |
595 | if (AsCall && SizeIndex < kNumberOfAccessSizes) { | |
596 | Value *Fn = MS.MaybeStoreOriginFn[SizeIndex]; | |
597 | Value *ConvertedShadow2 = IRB.CreateZExt( | |
598 | ConvertedShadow, IRB.getIntNTy(8 * (1 << SizeIndex))); | |
599 | IRB.CreateCall3(Fn, ConvertedShadow2, | |
600 | IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()), | |
601 | Origin); | |
602 | } else { | |
603 | Value *Cmp = IRB.CreateICmpNE( | |
604 | ConvertedShadow, getCleanShadow(ConvertedShadow), "_mscmp"); | |
605 | Instruction *CheckTerm = SplitBlockAndInsertIfThen( | |
606 | Cmp, IRB.GetInsertPoint(), false, MS.OriginStoreWeights); | |
607 | IRBuilder<> IRBNew(CheckTerm); | |
608 | IRBNew.CreateAlignedStore(updateOrigin(Origin, IRBNew), | |
85aaf69f SL |
609 | getOriginPtr(Addr, IRBNew, Alignment), |
610 | OriginAlignment); | |
1a4d82fc JJ |
611 | } |
612 | } | |
613 | } | |
614 | ||
615 | void materializeStores(bool InstrumentWithCalls) { | |
616 | for (auto Inst : StoreList) { | |
617 | StoreInst &SI = *dyn_cast<StoreInst>(Inst); | |
618 | ||
619 | IRBuilder<> IRB(&SI); | |
620 | Value *Val = SI.getValueOperand(); | |
621 | Value *Addr = SI.getPointerOperand(); | |
622 | Value *Shadow = SI.isAtomic() ? getCleanShadow(Val) : getShadow(Val); | |
970d7e83 LB |
623 | Value *ShadowPtr = getShadowPtr(Addr, Shadow->getType(), IRB); |
624 | ||
625 | StoreInst *NewSI = | |
1a4d82fc | 626 | IRB.CreateAlignedStore(Shadow, ShadowPtr, SI.getAlignment()); |
970d7e83 LB |
627 | DEBUG(dbgs() << " STORE: " << *NewSI << "\n"); |
628 | (void)NewSI; | |
629 | ||
1a4d82fc JJ |
630 | if (ClCheckAccessAddress) insertShadowCheck(Addr, &SI); |
631 | ||
632 | if (SI.isAtomic()) SI.setOrdering(addReleaseOrdering(SI.getOrdering())); | |
970d7e83 | 633 | |
85aaf69f SL |
634 | if (MS.TrackOrigins && !SI.isAtomic()) |
635 | storeOrigin(IRB, Addr, Shadow, getOrigin(Val), SI.getAlignment(), | |
1a4d82fc | 636 | InstrumentWithCalls); |
970d7e83 LB |
637 | } |
638 | } | |
639 | ||
1a4d82fc JJ |
640 | void materializeOneCheck(Instruction *OrigIns, Value *Shadow, Value *Origin, |
641 | bool AsCall) { | |
642 | IRBuilder<> IRB(OrigIns); | |
643 | DEBUG(dbgs() << " SHAD0 : " << *Shadow << "\n"); | |
644 | Value *ConvertedShadow = convertToShadowTyNoVec(Shadow, IRB); | |
645 | DEBUG(dbgs() << " SHAD1 : " << *ConvertedShadow << "\n"); | |
85aaf69f SL |
646 | // See the comment in storeOrigin(). |
647 | if (!ClCheckConstantShadow) | |
648 | if (isa<Constant>(ConvertedShadow)) return; | |
1a4d82fc JJ |
649 | unsigned TypeSizeInBits = |
650 | MS.DL->getTypeSizeInBits(ConvertedShadow->getType()); | |
651 | unsigned SizeIndex = TypeSizeToSizeIndex(TypeSizeInBits); | |
652 | if (AsCall && SizeIndex < kNumberOfAccessSizes) { | |
653 | Value *Fn = MS.MaybeWarningFn[SizeIndex]; | |
654 | Value *ConvertedShadow2 = | |
655 | IRB.CreateZExt(ConvertedShadow, IRB.getIntNTy(8 * (1 << SizeIndex))); | |
656 | IRB.CreateCall2(Fn, ConvertedShadow2, MS.TrackOrigins && Origin | |
657 | ? Origin | |
658 | : (Value *)IRB.getInt32(0)); | |
659 | } else { | |
970d7e83 LB |
660 | Value *Cmp = IRB.CreateICmpNE(ConvertedShadow, |
661 | getCleanShadow(ConvertedShadow), "_mscmp"); | |
1a4d82fc JJ |
662 | Instruction *CheckTerm = SplitBlockAndInsertIfThen( |
663 | Cmp, OrigIns, | |
664 | /* Unreachable */ !ClKeepGoing, MS.ColdCallWeights); | |
970d7e83 LB |
665 | |
666 | IRB.SetInsertPoint(CheckTerm); | |
667 | if (MS.TrackOrigins) { | |
1a4d82fc | 668 | IRB.CreateStore(Origin ? (Value *)Origin : (Value *)IRB.getInt32(0), |
970d7e83 LB |
669 | MS.OriginTLS); |
670 | } | |
1a4d82fc | 671 | IRB.CreateCall(MS.WarningFn); |
970d7e83 LB |
672 | IRB.CreateCall(MS.EmptyAsm); |
673 | DEBUG(dbgs() << " CHECK: " << *Cmp << "\n"); | |
674 | } | |
1a4d82fc JJ |
675 | } |
676 | ||
677 | void materializeChecks(bool InstrumentWithCalls) { | |
678 | for (const auto &ShadowData : InstrumentationList) { | |
679 | Instruction *OrigIns = ShadowData.OrigIns; | |
680 | Value *Shadow = ShadowData.Shadow; | |
681 | Value *Origin = ShadowData.Origin; | |
682 | materializeOneCheck(OrigIns, Shadow, Origin, InstrumentWithCalls); | |
683 | } | |
970d7e83 LB |
684 | DEBUG(dbgs() << "DONE:\n" << F); |
685 | } | |
686 | ||
687 | /// \brief Add MemorySanitizer instrumentation to a function. | |
688 | bool runOnFunction() { | |
689 | MS.initializeCallbacks(*F.getParent()); | |
1a4d82fc | 690 | if (!MS.DL) return false; |
970d7e83 LB |
691 | |
692 | // In the presence of unreachable blocks, we may see Phi nodes with | |
693 | // incoming nodes from such blocks. Since InstVisitor skips unreachable | |
694 | // blocks, such nodes will not have any shadow value associated with them. | |
695 | // It's easier to remove unreachable blocks than deal with missing shadow. | |
696 | removeUnreachableBlocks(F); | |
697 | ||
698 | // Iterate all BBs in depth-first order and create shadow instructions | |
699 | // for all instructions (where applicable). | |
700 | // For PHI nodes we create dummy shadow PHIs which will be finalized later. | |
1a4d82fc | 701 | for (BasicBlock *BB : depth_first(&F.getEntryBlock())) |
970d7e83 | 702 | visit(*BB); |
1a4d82fc | 703 | |
970d7e83 LB |
704 | |
705 | // Finalize PHI nodes. | |
1a4d82fc | 706 | for (PHINode *PN : ShadowPHINodes) { |
970d7e83 | 707 | PHINode *PNS = cast<PHINode>(getShadow(PN)); |
1a4d82fc | 708 | PHINode *PNO = MS.TrackOrigins ? cast<PHINode>(getOrigin(PN)) : nullptr; |
970d7e83 LB |
709 | size_t NumValues = PN->getNumIncomingValues(); |
710 | for (size_t v = 0; v < NumValues; v++) { | |
711 | PNS->addIncoming(getShadow(PN, v), PN->getIncomingBlock(v)); | |
1a4d82fc | 712 | if (PNO) PNO->addIncoming(getOrigin(PN, v), PN->getIncomingBlock(v)); |
970d7e83 LB |
713 | } |
714 | } | |
715 | ||
716 | VAHelper->finalizeInstrumentation(); | |
717 | ||
1a4d82fc JJ |
718 | bool InstrumentWithCalls = ClInstrumentationWithCallThreshold >= 0 && |
719 | InstrumentationList.size() + StoreList.size() > | |
720 | (unsigned)ClInstrumentationWithCallThreshold; | |
721 | ||
970d7e83 LB |
722 | // Delayed instrumentation of StoreInst. |
723 | // This may add new checks to be inserted later. | |
1a4d82fc | 724 | materializeStores(InstrumentWithCalls); |
970d7e83 LB |
725 | |
726 | // Insert shadow value checks. | |
1a4d82fc JJ |
727 | materializeChecks(InstrumentWithCalls); |
728 | ||
970d7e83 LB |
729 | return true; |
730 | } | |
731 | ||
732 | /// \brief Compute the shadow type that corresponds to a given Value. | |
733 | Type *getShadowTy(Value *V) { | |
734 | return getShadowTy(V->getType()); | |
735 | } | |
736 | ||
737 | /// \brief Compute the shadow type that corresponds to a given Type. | |
738 | Type *getShadowTy(Type *OrigTy) { | |
739 | if (!OrigTy->isSized()) { | |
1a4d82fc | 740 | return nullptr; |
970d7e83 LB |
741 | } |
742 | // For integer type, shadow is the same as the original type. | |
743 | // This may return weird-sized types like i1. | |
744 | if (IntegerType *IT = dyn_cast<IntegerType>(OrigTy)) | |
745 | return IT; | |
746 | if (VectorType *VT = dyn_cast<VectorType>(OrigTy)) { | |
1a4d82fc | 747 | uint32_t EltSize = MS.DL->getTypeSizeInBits(VT->getElementType()); |
970d7e83 LB |
748 | return VectorType::get(IntegerType::get(*MS.C, EltSize), |
749 | VT->getNumElements()); | |
750 | } | |
1a4d82fc JJ |
751 | if (ArrayType *AT = dyn_cast<ArrayType>(OrigTy)) { |
752 | return ArrayType::get(getShadowTy(AT->getElementType()), | |
753 | AT->getNumElements()); | |
754 | } | |
970d7e83 LB |
755 | if (StructType *ST = dyn_cast<StructType>(OrigTy)) { |
756 | SmallVector<Type*, 4> Elements; | |
757 | for (unsigned i = 0, n = ST->getNumElements(); i < n; i++) | |
758 | Elements.push_back(getShadowTy(ST->getElementType(i))); | |
759 | StructType *Res = StructType::get(*MS.C, Elements, ST->isPacked()); | |
760 | DEBUG(dbgs() << "getShadowTy: " << *ST << " ===> " << *Res << "\n"); | |
761 | return Res; | |
762 | } | |
1a4d82fc | 763 | uint32_t TypeSize = MS.DL->getTypeSizeInBits(OrigTy); |
970d7e83 LB |
764 | return IntegerType::get(*MS.C, TypeSize); |
765 | } | |
766 | ||
767 | /// \brief Flatten a vector type. | |
768 | Type *getShadowTyNoVec(Type *ty) { | |
769 | if (VectorType *vt = dyn_cast<VectorType>(ty)) | |
770 | return IntegerType::get(*MS.C, vt->getBitWidth()); | |
771 | return ty; | |
772 | } | |
773 | ||
774 | /// \brief Convert a shadow value to it's flattened variant. | |
775 | Value *convertToShadowTyNoVec(Value *V, IRBuilder<> &IRB) { | |
776 | Type *Ty = V->getType(); | |
777 | Type *NoVecTy = getShadowTyNoVec(Ty); | |
778 | if (Ty == NoVecTy) return V; | |
779 | return IRB.CreateBitCast(V, NoVecTy); | |
780 | } | |
781 | ||
85aaf69f SL |
782 | /// \brief Compute the integer shadow offset that corresponds to a given |
783 | /// application address. | |
784 | /// | |
785 | /// Offset = (Addr & ~AndMask) ^ XorMask | |
786 | Value *getShadowPtrOffset(Value *Addr, IRBuilder<> &IRB) { | |
787 | uint64_t AndMask = MS.MapParams->AndMask; | |
788 | assert(AndMask != 0 && "AndMask shall be specified"); | |
789 | Value *OffsetLong = | |
790 | IRB.CreateAnd(IRB.CreatePointerCast(Addr, MS.IntptrTy), | |
791 | ConstantInt::get(MS.IntptrTy, ~AndMask)); | |
792 | ||
793 | uint64_t XorMask = MS.MapParams->XorMask; | |
794 | if (XorMask != 0) | |
795 | OffsetLong = IRB.CreateXor(OffsetLong, | |
796 | ConstantInt::get(MS.IntptrTy, XorMask)); | |
797 | return OffsetLong; | |
798 | } | |
799 | ||
970d7e83 LB |
800 | /// \brief Compute the shadow address that corresponds to a given application |
801 | /// address. | |
802 | /// | |
85aaf69f | 803 | /// Shadow = ShadowBase + Offset |
970d7e83 LB |
804 | Value *getShadowPtr(Value *Addr, Type *ShadowTy, |
805 | IRBuilder<> &IRB) { | |
85aaf69f SL |
806 | Value *ShadowLong = getShadowPtrOffset(Addr, IRB); |
807 | uint64_t ShadowBase = MS.MapParams->ShadowBase; | |
808 | if (ShadowBase != 0) | |
809 | ShadowLong = | |
810 | IRB.CreateAdd(ShadowLong, | |
811 | ConstantInt::get(MS.IntptrTy, ShadowBase)); | |
970d7e83 LB |
812 | return IRB.CreateIntToPtr(ShadowLong, PointerType::get(ShadowTy, 0)); |
813 | } | |
814 | ||
815 | /// \brief Compute the origin address that corresponds to a given application | |
816 | /// address. | |
817 | /// | |
85aaf69f SL |
818 | /// OriginAddr = (OriginBase + Offset) & ~3ULL |
819 | Value *getOriginPtr(Value *Addr, IRBuilder<> &IRB, unsigned Alignment) { | |
820 | Value *OriginLong = getShadowPtrOffset(Addr, IRB); | |
821 | uint64_t OriginBase = MS.MapParams->OriginBase; | |
822 | if (OriginBase != 0) | |
823 | OriginLong = | |
824 | IRB.CreateAdd(OriginLong, | |
825 | ConstantInt::get(MS.IntptrTy, OriginBase)); | |
826 | if (Alignment < kMinOriginAlignment) { | |
827 | uint64_t Mask = kMinOriginAlignment - 1; | |
828 | OriginLong = IRB.CreateAnd(OriginLong, | |
829 | ConstantInt::get(MS.IntptrTy, ~Mask)); | |
830 | } | |
831 | return IRB.CreateIntToPtr(OriginLong, | |
832 | PointerType::get(IRB.getInt32Ty(), 0)); | |
970d7e83 LB |
833 | } |
834 | ||
835 | /// \brief Compute the shadow address for a given function argument. | |
836 | /// | |
837 | /// Shadow = ParamTLS+ArgOffset. | |
838 | Value *getShadowPtrForArgument(Value *A, IRBuilder<> &IRB, | |
839 | int ArgOffset) { | |
840 | Value *Base = IRB.CreatePointerCast(MS.ParamTLS, MS.IntptrTy); | |
841 | Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset)); | |
842 | return IRB.CreateIntToPtr(Base, PointerType::get(getShadowTy(A), 0), | |
843 | "_msarg"); | |
844 | } | |
845 | ||
846 | /// \brief Compute the origin address for a given function argument. | |
847 | Value *getOriginPtrForArgument(Value *A, IRBuilder<> &IRB, | |
848 | int ArgOffset) { | |
1a4d82fc | 849 | if (!MS.TrackOrigins) return nullptr; |
970d7e83 LB |
850 | Value *Base = IRB.CreatePointerCast(MS.ParamOriginTLS, MS.IntptrTy); |
851 | Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset)); | |
852 | return IRB.CreateIntToPtr(Base, PointerType::get(MS.OriginTy, 0), | |
853 | "_msarg_o"); | |
854 | } | |
855 | ||
856 | /// \brief Compute the shadow address for a retval. | |
857 | Value *getShadowPtrForRetval(Value *A, IRBuilder<> &IRB) { | |
858 | Value *Base = IRB.CreatePointerCast(MS.RetvalTLS, MS.IntptrTy); | |
859 | return IRB.CreateIntToPtr(Base, PointerType::get(getShadowTy(A), 0), | |
860 | "_msret"); | |
861 | } | |
862 | ||
863 | /// \brief Compute the origin address for a retval. | |
864 | Value *getOriginPtrForRetval(IRBuilder<> &IRB) { | |
865 | // We keep a single origin for the entire retval. Might be too optimistic. | |
866 | return MS.RetvalOriginTLS; | |
867 | } | |
868 | ||
869 | /// \brief Set SV to be the shadow value for V. | |
870 | void setShadow(Value *V, Value *SV) { | |
871 | assert(!ShadowMap.count(V) && "Values may only have one shadow"); | |
1a4d82fc | 872 | ShadowMap[V] = PropagateShadow ? SV : getCleanShadow(V); |
970d7e83 LB |
873 | } |
874 | ||
875 | /// \brief Set Origin to be the origin value for V. | |
876 | void setOrigin(Value *V, Value *Origin) { | |
877 | if (!MS.TrackOrigins) return; | |
878 | assert(!OriginMap.count(V) && "Values may only have one origin"); | |
879 | DEBUG(dbgs() << "ORIGIN: " << *V << " ==> " << *Origin << "\n"); | |
880 | OriginMap[V] = Origin; | |
881 | } | |
882 | ||
883 | /// \brief Create a clean shadow value for a given value. | |
884 | /// | |
885 | /// Clean shadow (all zeroes) means all bits of the value are defined | |
886 | /// (initialized). | |
1a4d82fc | 887 | Constant *getCleanShadow(Value *V) { |
970d7e83 LB |
888 | Type *ShadowTy = getShadowTy(V); |
889 | if (!ShadowTy) | |
1a4d82fc | 890 | return nullptr; |
970d7e83 LB |
891 | return Constant::getNullValue(ShadowTy); |
892 | } | |
893 | ||
894 | /// \brief Create a dirty shadow of a given shadow type. | |
895 | Constant *getPoisonedShadow(Type *ShadowTy) { | |
896 | assert(ShadowTy); | |
897 | if (isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy)) | |
898 | return Constant::getAllOnesValue(ShadowTy); | |
1a4d82fc JJ |
899 | if (ArrayType *AT = dyn_cast<ArrayType>(ShadowTy)) { |
900 | SmallVector<Constant *, 4> Vals(AT->getNumElements(), | |
901 | getPoisonedShadow(AT->getElementType())); | |
902 | return ConstantArray::get(AT, Vals); | |
903 | } | |
904 | if (StructType *ST = dyn_cast<StructType>(ShadowTy)) { | |
905 | SmallVector<Constant *, 4> Vals; | |
906 | for (unsigned i = 0, n = ST->getNumElements(); i < n; i++) | |
907 | Vals.push_back(getPoisonedShadow(ST->getElementType(i))); | |
908 | return ConstantStruct::get(ST, Vals); | |
909 | } | |
910 | llvm_unreachable("Unexpected shadow type"); | |
911 | } | |
912 | ||
913 | /// \brief Create a dirty shadow for a given value. | |
914 | Constant *getPoisonedShadow(Value *V) { | |
915 | Type *ShadowTy = getShadowTy(V); | |
916 | if (!ShadowTy) | |
917 | return nullptr; | |
918 | return getPoisonedShadow(ShadowTy); | |
970d7e83 LB |
919 | } |
920 | ||
921 | /// \brief Create a clean (zero) origin. | |
922 | Value *getCleanOrigin() { | |
923 | return Constant::getNullValue(MS.OriginTy); | |
924 | } | |
925 | ||
926 | /// \brief Get the shadow value for a given Value. | |
927 | /// | |
928 | /// This function either returns the value set earlier with setShadow, | |
929 | /// or extracts if from ParamTLS (for function arguments). | |
930 | Value *getShadow(Value *V) { | |
1a4d82fc | 931 | if (!PropagateShadow) return getCleanShadow(V); |
970d7e83 LB |
932 | if (Instruction *I = dyn_cast<Instruction>(V)) { |
933 | // For instructions the shadow is already stored in the map. | |
934 | Value *Shadow = ShadowMap[V]; | |
935 | if (!Shadow) { | |
936 | DEBUG(dbgs() << "No shadow: " << *V << "\n" << *(I->getParent())); | |
937 | (void)I; | |
938 | assert(Shadow && "No shadow for a value"); | |
939 | } | |
940 | return Shadow; | |
941 | } | |
942 | if (UndefValue *U = dyn_cast<UndefValue>(V)) { | |
1a4d82fc | 943 | Value *AllOnes = PoisonUndef ? getPoisonedShadow(V) : getCleanShadow(V); |
970d7e83 LB |
944 | DEBUG(dbgs() << "Undef: " << *U << " ==> " << *AllOnes << "\n"); |
945 | (void)U; | |
946 | return AllOnes; | |
947 | } | |
948 | if (Argument *A = dyn_cast<Argument>(V)) { | |
949 | // For arguments we compute the shadow on demand and store it in the map. | |
950 | Value **ShadowPtr = &ShadowMap[V]; | |
951 | if (*ShadowPtr) | |
952 | return *ShadowPtr; | |
953 | Function *F = A->getParent(); | |
954 | IRBuilder<> EntryIRB(F->getEntryBlock().getFirstNonPHI()); | |
955 | unsigned ArgOffset = 0; | |
1a4d82fc JJ |
956 | for (auto &FArg : F->args()) { |
957 | if (!FArg.getType()->isSized()) { | |
970d7e83 LB |
958 | DEBUG(dbgs() << "Arg is not sized\n"); |
959 | continue; | |
960 | } | |
1a4d82fc JJ |
961 | unsigned Size = FArg.hasByValAttr() |
962 | ? MS.DL->getTypeAllocSize(FArg.getType()->getPointerElementType()) | |
963 | : MS.DL->getTypeAllocSize(FArg.getType()); | |
964 | if (A == &FArg) { | |
85aaf69f | 965 | bool Overflow = ArgOffset + Size > kParamTLSSize; |
1a4d82fc JJ |
966 | Value *Base = getShadowPtrForArgument(&FArg, EntryIRB, ArgOffset); |
967 | if (FArg.hasByValAttr()) { | |
970d7e83 LB |
968 | // ByVal pointer itself has clean shadow. We copy the actual |
969 | // argument shadow to the underlying memory. | |
1a4d82fc JJ |
970 | // Figure out maximal valid memcpy alignment. |
971 | unsigned ArgAlign = FArg.getParamAlignment(); | |
972 | if (ArgAlign == 0) { | |
973 | Type *EltType = A->getType()->getPointerElementType(); | |
974 | ArgAlign = MS.DL->getABITypeAlignment(EltType); | |
975 | } | |
85aaf69f SL |
976 | if (Overflow) { |
977 | // ParamTLS overflow. | |
978 | EntryIRB.CreateMemSet( | |
979 | getShadowPtr(V, EntryIRB.getInt8Ty(), EntryIRB), | |
980 | Constant::getNullValue(EntryIRB.getInt8Ty()), Size, ArgAlign); | |
981 | } else { | |
982 | unsigned CopyAlign = std::min(ArgAlign, kShadowTLSAlignment); | |
983 | Value *Cpy = EntryIRB.CreateMemCpy( | |
984 | getShadowPtr(V, EntryIRB.getInt8Ty(), EntryIRB), Base, Size, | |
985 | CopyAlign); | |
986 | DEBUG(dbgs() << " ByValCpy: " << *Cpy << "\n"); | |
987 | (void)Cpy; | |
988 | } | |
970d7e83 LB |
989 | *ShadowPtr = getCleanShadow(V); |
990 | } else { | |
85aaf69f SL |
991 | if (Overflow) { |
992 | // ParamTLS overflow. | |
993 | *ShadowPtr = getCleanShadow(V); | |
994 | } else { | |
995 | *ShadowPtr = | |
996 | EntryIRB.CreateAlignedLoad(Base, kShadowTLSAlignment); | |
997 | } | |
970d7e83 | 998 | } |
1a4d82fc | 999 | DEBUG(dbgs() << " ARG: " << FArg << " ==> " << |
970d7e83 | 1000 | **ShadowPtr << "\n"); |
85aaf69f | 1001 | if (MS.TrackOrigins && !Overflow) { |
1a4d82fc JJ |
1002 | Value *OriginPtr = |
1003 | getOriginPtrForArgument(&FArg, EntryIRB, ArgOffset); | |
970d7e83 | 1004 | setOrigin(A, EntryIRB.CreateLoad(OriginPtr)); |
85aaf69f SL |
1005 | } else { |
1006 | setOrigin(A, getCleanOrigin()); | |
970d7e83 LB |
1007 | } |
1008 | } | |
85aaf69f | 1009 | ArgOffset += RoundUpToAlignment(Size, kShadowTLSAlignment); |
970d7e83 LB |
1010 | } |
1011 | assert(*ShadowPtr && "Could not find shadow for an argument"); | |
1012 | return *ShadowPtr; | |
1013 | } | |
1014 | // For everything else the shadow is zero. | |
1015 | return getCleanShadow(V); | |
1016 | } | |
1017 | ||
1018 | /// \brief Get the shadow for i-th argument of the instruction I. | |
1019 | Value *getShadow(Instruction *I, int i) { | |
1020 | return getShadow(I->getOperand(i)); | |
1021 | } | |
1022 | ||
1023 | /// \brief Get the origin for a value. | |
1024 | Value *getOrigin(Value *V) { | |
1a4d82fc | 1025 | if (!MS.TrackOrigins) return nullptr; |
85aaf69f SL |
1026 | if (!PropagateShadow) return getCleanOrigin(); |
1027 | if (isa<Constant>(V)) return getCleanOrigin(); | |
1028 | assert((isa<Instruction>(V) || isa<Argument>(V)) && | |
1029 | "Unexpected value type in getOrigin()"); | |
1030 | Value *Origin = OriginMap[V]; | |
1031 | assert(Origin && "Missing origin"); | |
1032 | return Origin; | |
970d7e83 LB |
1033 | } |
1034 | ||
1035 | /// \brief Get the origin for i-th argument of the instruction I. | |
1036 | Value *getOrigin(Instruction *I, int i) { | |
1037 | return getOrigin(I->getOperand(i)); | |
1038 | } | |
1039 | ||
1040 | /// \brief Remember the place where a shadow check should be inserted. | |
1041 | /// | |
1042 | /// This location will be later instrumented with a check that will print a | |
1a4d82fc JJ |
1043 | /// UMR warning in runtime if the shadow value is not 0. |
1044 | void insertShadowCheck(Value *Shadow, Value *Origin, Instruction *OrigIns) { | |
1045 | assert(Shadow); | |
970d7e83 | 1046 | if (!InsertChecks) return; |
970d7e83 LB |
1047 | #ifndef NDEBUG |
1048 | Type *ShadowTy = Shadow->getType(); | |
1049 | assert((isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy)) && | |
1050 | "Can only insert checks for integer and vector shadow types"); | |
1051 | #endif | |
970d7e83 | 1052 | InstrumentationList.push_back( |
1a4d82fc JJ |
1053 | ShadowOriginAndInsertPoint(Shadow, Origin, OrigIns)); |
1054 | } | |
1055 | ||
1056 | /// \brief Remember the place where a shadow check should be inserted. | |
1057 | /// | |
1058 | /// This location will be later instrumented with a check that will print a | |
1059 | /// UMR warning in runtime if the value is not fully defined. | |
1060 | void insertShadowCheck(Value *Val, Instruction *OrigIns) { | |
1061 | assert(Val); | |
85aaf69f SL |
1062 | Value *Shadow, *Origin; |
1063 | if (ClCheckConstantShadow) { | |
1064 | Shadow = getShadow(Val); | |
1065 | if (!Shadow) return; | |
1066 | Origin = getOrigin(Val); | |
1067 | } else { | |
1068 | Shadow = dyn_cast_or_null<Instruction>(getShadow(Val)); | |
1069 | if (!Shadow) return; | |
1070 | Origin = dyn_cast_or_null<Instruction>(getOrigin(Val)); | |
1071 | } | |
1a4d82fc JJ |
1072 | insertShadowCheck(Shadow, Origin, OrigIns); |
1073 | } | |
1074 | ||
1075 | AtomicOrdering addReleaseOrdering(AtomicOrdering a) { | |
1076 | switch (a) { | |
1077 | case NotAtomic: | |
1078 | return NotAtomic; | |
1079 | case Unordered: | |
1080 | case Monotonic: | |
1081 | case Release: | |
1082 | return Release; | |
1083 | case Acquire: | |
1084 | case AcquireRelease: | |
1085 | return AcquireRelease; | |
1086 | case SequentiallyConsistent: | |
1087 | return SequentiallyConsistent; | |
1088 | } | |
1089 | llvm_unreachable("Unknown ordering"); | |
1090 | } | |
1091 | ||
1092 | AtomicOrdering addAcquireOrdering(AtomicOrdering a) { | |
1093 | switch (a) { | |
1094 | case NotAtomic: | |
1095 | return NotAtomic; | |
1096 | case Unordered: | |
1097 | case Monotonic: | |
1098 | case Acquire: | |
1099 | return Acquire; | |
1100 | case Release: | |
1101 | case AcquireRelease: | |
1102 | return AcquireRelease; | |
1103 | case SequentiallyConsistent: | |
1104 | return SequentiallyConsistent; | |
1105 | } | |
1106 | llvm_unreachable("Unknown ordering"); | |
970d7e83 LB |
1107 | } |
1108 | ||
1109 | // ------------------- Visitors. | |
1110 | ||
1111 | /// \brief Instrument LoadInst | |
1112 | /// | |
1113 | /// Loads the corresponding shadow and (optionally) origin. | |
1114 | /// Optionally, checks that the load address is fully defined. | |
1115 | void visitLoadInst(LoadInst &I) { | |
1116 | assert(I.getType()->isSized() && "Load type must have size"); | |
1a4d82fc | 1117 | IRBuilder<> IRB(I.getNextNode()); |
970d7e83 LB |
1118 | Type *ShadowTy = getShadowTy(&I); |
1119 | Value *Addr = I.getPointerOperand(); | |
85aaf69f | 1120 | if (PropagateShadow && !I.getMetadata("nosanitize")) { |
970d7e83 LB |
1121 | Value *ShadowPtr = getShadowPtr(Addr, ShadowTy, IRB); |
1122 | setShadow(&I, | |
1123 | IRB.CreateAlignedLoad(ShadowPtr, I.getAlignment(), "_msld")); | |
1124 | } else { | |
1125 | setShadow(&I, getCleanShadow(&I)); | |
1126 | } | |
1127 | ||
1128 | if (ClCheckAccessAddress) | |
1a4d82fc JJ |
1129 | insertShadowCheck(I.getPointerOperand(), &I); |
1130 | ||
1131 | if (I.isAtomic()) | |
1132 | I.setOrdering(addAcquireOrdering(I.getOrdering())); | |
970d7e83 LB |
1133 | |
1134 | if (MS.TrackOrigins) { | |
1a4d82fc | 1135 | if (PropagateShadow) { |
85aaf69f SL |
1136 | unsigned Alignment = I.getAlignment(); |
1137 | unsigned OriginAlignment = std::max(kMinOriginAlignment, Alignment); | |
1138 | setOrigin(&I, IRB.CreateAlignedLoad(getOriginPtr(Addr, IRB, Alignment), | |
1139 | OriginAlignment)); | |
970d7e83 LB |
1140 | } else { |
1141 | setOrigin(&I, getCleanOrigin()); | |
1142 | } | |
1143 | } | |
1144 | } | |
1145 | ||
1146 | /// \brief Instrument StoreInst | |
1147 | /// | |
1148 | /// Stores the corresponding shadow and (optionally) origin. | |
1149 | /// Optionally, checks that the store address is fully defined. | |
1150 | void visitStoreInst(StoreInst &I) { | |
1151 | StoreList.push_back(&I); | |
1152 | } | |
1153 | ||
1a4d82fc JJ |
1154 | void handleCASOrRMW(Instruction &I) { |
1155 | assert(isa<AtomicRMWInst>(I) || isa<AtomicCmpXchgInst>(I)); | |
1156 | ||
1157 | IRBuilder<> IRB(&I); | |
1158 | Value *Addr = I.getOperand(0); | |
1159 | Value *ShadowPtr = getShadowPtr(Addr, I.getType(), IRB); | |
1160 | ||
1161 | if (ClCheckAccessAddress) | |
1162 | insertShadowCheck(Addr, &I); | |
1163 | ||
1164 | // Only test the conditional argument of cmpxchg instruction. | |
1165 | // The other argument can potentially be uninitialized, but we can not | |
1166 | // detect this situation reliably without possible false positives. | |
1167 | if (isa<AtomicCmpXchgInst>(I)) | |
1168 | insertShadowCheck(I.getOperand(1), &I); | |
1169 | ||
1170 | IRB.CreateStore(getCleanShadow(&I), ShadowPtr); | |
1171 | ||
1172 | setShadow(&I, getCleanShadow(&I)); | |
85aaf69f | 1173 | setOrigin(&I, getCleanOrigin()); |
1a4d82fc JJ |
1174 | } |
1175 | ||
1176 | void visitAtomicRMWInst(AtomicRMWInst &I) { | |
1177 | handleCASOrRMW(I); | |
1178 | I.setOrdering(addReleaseOrdering(I.getOrdering())); | |
1179 | } | |
1180 | ||
1181 | void visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) { | |
1182 | handleCASOrRMW(I); | |
1183 | I.setSuccessOrdering(addReleaseOrdering(I.getSuccessOrdering())); | |
1184 | } | |
1185 | ||
970d7e83 LB |
1186 | // Vector manipulation. |
1187 | void visitExtractElementInst(ExtractElementInst &I) { | |
1a4d82fc | 1188 | insertShadowCheck(I.getOperand(1), &I); |
970d7e83 LB |
1189 | IRBuilder<> IRB(&I); |
1190 | setShadow(&I, IRB.CreateExtractElement(getShadow(&I, 0), I.getOperand(1), | |
1191 | "_msprop")); | |
1192 | setOrigin(&I, getOrigin(&I, 0)); | |
1193 | } | |
1194 | ||
1195 | void visitInsertElementInst(InsertElementInst &I) { | |
1a4d82fc | 1196 | insertShadowCheck(I.getOperand(2), &I); |
970d7e83 LB |
1197 | IRBuilder<> IRB(&I); |
1198 | setShadow(&I, IRB.CreateInsertElement(getShadow(&I, 0), getShadow(&I, 1), | |
1199 | I.getOperand(2), "_msprop")); | |
1200 | setOriginForNaryOp(I); | |
1201 | } | |
1202 | ||
1203 | void visitShuffleVectorInst(ShuffleVectorInst &I) { | |
1a4d82fc | 1204 | insertShadowCheck(I.getOperand(2), &I); |
970d7e83 LB |
1205 | IRBuilder<> IRB(&I); |
1206 | setShadow(&I, IRB.CreateShuffleVector(getShadow(&I, 0), getShadow(&I, 1), | |
1207 | I.getOperand(2), "_msprop")); | |
1208 | setOriginForNaryOp(I); | |
1209 | } | |
1210 | ||
1211 | // Casts. | |
1212 | void visitSExtInst(SExtInst &I) { | |
1213 | IRBuilder<> IRB(&I); | |
1214 | setShadow(&I, IRB.CreateSExt(getShadow(&I, 0), I.getType(), "_msprop")); | |
1215 | setOrigin(&I, getOrigin(&I, 0)); | |
1216 | } | |
1217 | ||
1218 | void visitZExtInst(ZExtInst &I) { | |
1219 | IRBuilder<> IRB(&I); | |
1220 | setShadow(&I, IRB.CreateZExt(getShadow(&I, 0), I.getType(), "_msprop")); | |
1221 | setOrigin(&I, getOrigin(&I, 0)); | |
1222 | } | |
1223 | ||
1224 | void visitTruncInst(TruncInst &I) { | |
1225 | IRBuilder<> IRB(&I); | |
1226 | setShadow(&I, IRB.CreateTrunc(getShadow(&I, 0), I.getType(), "_msprop")); | |
1227 | setOrigin(&I, getOrigin(&I, 0)); | |
1228 | } | |
1229 | ||
1230 | void visitBitCastInst(BitCastInst &I) { | |
1231 | IRBuilder<> IRB(&I); | |
1232 | setShadow(&I, IRB.CreateBitCast(getShadow(&I, 0), getShadowTy(&I))); | |
1233 | setOrigin(&I, getOrigin(&I, 0)); | |
1234 | } | |
1235 | ||
1236 | void visitPtrToIntInst(PtrToIntInst &I) { | |
1237 | IRBuilder<> IRB(&I); | |
1238 | setShadow(&I, IRB.CreateIntCast(getShadow(&I, 0), getShadowTy(&I), false, | |
1239 | "_msprop_ptrtoint")); | |
1240 | setOrigin(&I, getOrigin(&I, 0)); | |
1241 | } | |
1242 | ||
1243 | void visitIntToPtrInst(IntToPtrInst &I) { | |
1244 | IRBuilder<> IRB(&I); | |
1245 | setShadow(&I, IRB.CreateIntCast(getShadow(&I, 0), getShadowTy(&I), false, | |
1246 | "_msprop_inttoptr")); | |
1247 | setOrigin(&I, getOrigin(&I, 0)); | |
1248 | } | |
1249 | ||
1250 | void visitFPToSIInst(CastInst& I) { handleShadowOr(I); } | |
1251 | void visitFPToUIInst(CastInst& I) { handleShadowOr(I); } | |
1252 | void visitSIToFPInst(CastInst& I) { handleShadowOr(I); } | |
1253 | void visitUIToFPInst(CastInst& I) { handleShadowOr(I); } | |
1254 | void visitFPExtInst(CastInst& I) { handleShadowOr(I); } | |
1255 | void visitFPTruncInst(CastInst& I) { handleShadowOr(I); } | |
1256 | ||
1257 | /// \brief Propagate shadow for bitwise AND. | |
1258 | /// | |
1259 | /// This code is exact, i.e. if, for example, a bit in the left argument | |
1260 | /// is defined and 0, then neither the value not definedness of the | |
1261 | /// corresponding bit in B don't affect the resulting shadow. | |
1262 | void visitAnd(BinaryOperator &I) { | |
1263 | IRBuilder<> IRB(&I); | |
1264 | // "And" of 0 and a poisoned value results in unpoisoned value. | |
1265 | // 1&1 => 1; 0&1 => 0; p&1 => p; | |
1266 | // 1&0 => 0; 0&0 => 0; p&0 => 0; | |
1267 | // 1&p => p; 0&p => 0; p&p => p; | |
1268 | // S = (S1 & S2) | (V1 & S2) | (S1 & V2) | |
1269 | Value *S1 = getShadow(&I, 0); | |
1270 | Value *S2 = getShadow(&I, 1); | |
1271 | Value *V1 = I.getOperand(0); | |
1272 | Value *V2 = I.getOperand(1); | |
1273 | if (V1->getType() != S1->getType()) { | |
1274 | V1 = IRB.CreateIntCast(V1, S1->getType(), false); | |
1275 | V2 = IRB.CreateIntCast(V2, S2->getType(), false); | |
1276 | } | |
1277 | Value *S1S2 = IRB.CreateAnd(S1, S2); | |
1278 | Value *V1S2 = IRB.CreateAnd(V1, S2); | |
1279 | Value *S1V2 = IRB.CreateAnd(S1, V2); | |
1280 | setShadow(&I, IRB.CreateOr(S1S2, IRB.CreateOr(V1S2, S1V2))); | |
1281 | setOriginForNaryOp(I); | |
1282 | } | |
1283 | ||
1284 | void visitOr(BinaryOperator &I) { | |
1285 | IRBuilder<> IRB(&I); | |
1286 | // "Or" of 1 and a poisoned value results in unpoisoned value. | |
1287 | // 1|1 => 1; 0|1 => 1; p|1 => 1; | |
1288 | // 1|0 => 1; 0|0 => 0; p|0 => p; | |
1289 | // 1|p => 1; 0|p => p; p|p => p; | |
1290 | // S = (S1 & S2) | (~V1 & S2) | (S1 & ~V2) | |
1291 | Value *S1 = getShadow(&I, 0); | |
1292 | Value *S2 = getShadow(&I, 1); | |
1293 | Value *V1 = IRB.CreateNot(I.getOperand(0)); | |
1294 | Value *V2 = IRB.CreateNot(I.getOperand(1)); | |
1295 | if (V1->getType() != S1->getType()) { | |
1296 | V1 = IRB.CreateIntCast(V1, S1->getType(), false); | |
1297 | V2 = IRB.CreateIntCast(V2, S2->getType(), false); | |
1298 | } | |
1299 | Value *S1S2 = IRB.CreateAnd(S1, S2); | |
1300 | Value *V1S2 = IRB.CreateAnd(V1, S2); | |
1301 | Value *S1V2 = IRB.CreateAnd(S1, V2); | |
1302 | setShadow(&I, IRB.CreateOr(S1S2, IRB.CreateOr(V1S2, S1V2))); | |
1303 | setOriginForNaryOp(I); | |
1304 | } | |
1305 | ||
1306 | /// \brief Default propagation of shadow and/or origin. | |
1307 | /// | |
1308 | /// This class implements the general case of shadow propagation, used in all | |
1309 | /// cases where we don't know and/or don't care about what the operation | |
1310 | /// actually does. It converts all input shadow values to a common type | |
1311 | /// (extending or truncating as necessary), and bitwise OR's them. | |
1312 | /// | |
1313 | /// This is much cheaper than inserting checks (i.e. requiring inputs to be | |
1314 | /// fully initialized), and less prone to false positives. | |
1315 | /// | |
1316 | /// This class also implements the general case of origin propagation. For a | |
1317 | /// Nary operation, result origin is set to the origin of an argument that is | |
1318 | /// not entirely initialized. If there is more than one such arguments, the | |
1319 | /// rightmost of them is picked. It does not matter which one is picked if all | |
1320 | /// arguments are initialized. | |
1321 | template <bool CombineShadow> | |
1322 | class Combiner { | |
1323 | Value *Shadow; | |
1324 | Value *Origin; | |
1325 | IRBuilder<> &IRB; | |
1326 | MemorySanitizerVisitor *MSV; | |
1327 | ||
1328 | public: | |
1329 | Combiner(MemorySanitizerVisitor *MSV, IRBuilder<> &IRB) : | |
1a4d82fc | 1330 | Shadow(nullptr), Origin(nullptr), IRB(IRB), MSV(MSV) {} |
970d7e83 LB |
1331 | |
1332 | /// \brief Add a pair of shadow and origin values to the mix. | |
1333 | Combiner &Add(Value *OpShadow, Value *OpOrigin) { | |
1334 | if (CombineShadow) { | |
1335 | assert(OpShadow); | |
1336 | if (!Shadow) | |
1337 | Shadow = OpShadow; | |
1338 | else { | |
1339 | OpShadow = MSV->CreateShadowCast(IRB, OpShadow, Shadow->getType()); | |
1340 | Shadow = IRB.CreateOr(Shadow, OpShadow, "_msprop"); | |
1341 | } | |
1342 | } | |
1343 | ||
1344 | if (MSV->MS.TrackOrigins) { | |
1345 | assert(OpOrigin); | |
1346 | if (!Origin) { | |
1347 | Origin = OpOrigin; | |
1348 | } else { | |
1a4d82fc JJ |
1349 | Constant *ConstOrigin = dyn_cast<Constant>(OpOrigin); |
1350 | // No point in adding something that might result in 0 origin value. | |
1351 | if (!ConstOrigin || !ConstOrigin->isNullValue()) { | |
1352 | Value *FlatShadow = MSV->convertToShadowTyNoVec(OpShadow, IRB); | |
1353 | Value *Cond = | |
1354 | IRB.CreateICmpNE(FlatShadow, MSV->getCleanShadow(FlatShadow)); | |
1355 | Origin = IRB.CreateSelect(Cond, OpOrigin, Origin); | |
1356 | } | |
970d7e83 LB |
1357 | } |
1358 | } | |
1359 | return *this; | |
1360 | } | |
1361 | ||
1362 | /// \brief Add an application value to the mix. | |
1363 | Combiner &Add(Value *V) { | |
1364 | Value *OpShadow = MSV->getShadow(V); | |
1a4d82fc | 1365 | Value *OpOrigin = MSV->MS.TrackOrigins ? MSV->getOrigin(V) : nullptr; |
970d7e83 LB |
1366 | return Add(OpShadow, OpOrigin); |
1367 | } | |
1368 | ||
1369 | /// \brief Set the current combined values as the given instruction's shadow | |
1370 | /// and origin. | |
1371 | void Done(Instruction *I) { | |
1372 | if (CombineShadow) { | |
1373 | assert(Shadow); | |
1374 | Shadow = MSV->CreateShadowCast(IRB, Shadow, MSV->getShadowTy(I)); | |
1375 | MSV->setShadow(I, Shadow); | |
1376 | } | |
1377 | if (MSV->MS.TrackOrigins) { | |
1378 | assert(Origin); | |
1379 | MSV->setOrigin(I, Origin); | |
1380 | } | |
1381 | } | |
1382 | }; | |
1383 | ||
1384 | typedef Combiner<true> ShadowAndOriginCombiner; | |
1385 | typedef Combiner<false> OriginCombiner; | |
1386 | ||
1387 | /// \brief Propagate origin for arbitrary operation. | |
1388 | void setOriginForNaryOp(Instruction &I) { | |
1389 | if (!MS.TrackOrigins) return; | |
1390 | IRBuilder<> IRB(&I); | |
1391 | OriginCombiner OC(this, IRB); | |
1392 | for (Instruction::op_iterator OI = I.op_begin(); OI != I.op_end(); ++OI) | |
1393 | OC.Add(OI->get()); | |
1394 | OC.Done(&I); | |
1395 | } | |
1396 | ||
1397 | size_t VectorOrPrimitiveTypeSizeInBits(Type *Ty) { | |
1398 | assert(!(Ty->isVectorTy() && Ty->getScalarType()->isPointerTy()) && | |
1399 | "Vector of pointers is not a valid shadow type"); | |
1400 | return Ty->isVectorTy() ? | |
1401 | Ty->getVectorNumElements() * Ty->getScalarSizeInBits() : | |
1402 | Ty->getPrimitiveSizeInBits(); | |
1403 | } | |
1404 | ||
1405 | /// \brief Cast between two shadow types, extending or truncating as | |
1406 | /// necessary. | |
1a4d82fc JJ |
1407 | Value *CreateShadowCast(IRBuilder<> &IRB, Value *V, Type *dstTy, |
1408 | bool Signed = false) { | |
970d7e83 LB |
1409 | Type *srcTy = V->getType(); |
1410 | if (dstTy->isIntegerTy() && srcTy->isIntegerTy()) | |
1a4d82fc | 1411 | return IRB.CreateIntCast(V, dstTy, Signed); |
970d7e83 LB |
1412 | if (dstTy->isVectorTy() && srcTy->isVectorTy() && |
1413 | dstTy->getVectorNumElements() == srcTy->getVectorNumElements()) | |
1a4d82fc | 1414 | return IRB.CreateIntCast(V, dstTy, Signed); |
970d7e83 LB |
1415 | size_t srcSizeInBits = VectorOrPrimitiveTypeSizeInBits(srcTy); |
1416 | size_t dstSizeInBits = VectorOrPrimitiveTypeSizeInBits(dstTy); | |
1417 | Value *V1 = IRB.CreateBitCast(V, Type::getIntNTy(*MS.C, srcSizeInBits)); | |
1418 | Value *V2 = | |
1a4d82fc | 1419 | IRB.CreateIntCast(V1, Type::getIntNTy(*MS.C, dstSizeInBits), Signed); |
970d7e83 LB |
1420 | return IRB.CreateBitCast(V2, dstTy); |
1421 | // TODO: handle struct types. | |
1422 | } | |
1423 | ||
1a4d82fc JJ |
1424 | /// \brief Cast an application value to the type of its own shadow. |
1425 | Value *CreateAppToShadowCast(IRBuilder<> &IRB, Value *V) { | |
1426 | Type *ShadowTy = getShadowTy(V); | |
1427 | if (V->getType() == ShadowTy) | |
1428 | return V; | |
1429 | if (V->getType()->isPtrOrPtrVectorTy()) | |
1430 | return IRB.CreatePtrToInt(V, ShadowTy); | |
1431 | else | |
1432 | return IRB.CreateBitCast(V, ShadowTy); | |
1433 | } | |
1434 | ||
970d7e83 LB |
1435 | /// \brief Propagate shadow for arbitrary operation. |
1436 | void handleShadowOr(Instruction &I) { | |
1437 | IRBuilder<> IRB(&I); | |
1438 | ShadowAndOriginCombiner SC(this, IRB); | |
1439 | for (Instruction::op_iterator OI = I.op_begin(); OI != I.op_end(); ++OI) | |
1440 | SC.Add(OI->get()); | |
1441 | SC.Done(&I); | |
1442 | } | |
1443 | ||
1a4d82fc JJ |
1444 | // \brief Handle multiplication by constant. |
1445 | // | |
1446 | // Handle a special case of multiplication by constant that may have one or | |
1447 | // more zeros in the lower bits. This makes corresponding number of lower bits | |
1448 | // of the result zero as well. We model it by shifting the other operand | |
1449 | // shadow left by the required number of bits. Effectively, we transform | |
1450 | // (X * (A * 2**B)) to ((X << B) * A) and instrument (X << B) as (Sx << B). | |
1451 | // We use multiplication by 2**N instead of shift to cover the case of | |
1452 | // multiplication by 0, which may occur in some elements of a vector operand. | |
1453 | void handleMulByConstant(BinaryOperator &I, Constant *ConstArg, | |
1454 | Value *OtherArg) { | |
1455 | Constant *ShadowMul; | |
1456 | Type *Ty = ConstArg->getType(); | |
1457 | if (Ty->isVectorTy()) { | |
1458 | unsigned NumElements = Ty->getVectorNumElements(); | |
1459 | Type *EltTy = Ty->getSequentialElementType(); | |
1460 | SmallVector<Constant *, 16> Elements; | |
1461 | for (unsigned Idx = 0; Idx < NumElements; ++Idx) { | |
1462 | ConstantInt *Elt = | |
1463 | dyn_cast<ConstantInt>(ConstArg->getAggregateElement(Idx)); | |
1464 | APInt V = Elt->getValue(); | |
1465 | APInt V2 = APInt(V.getBitWidth(), 1) << V.countTrailingZeros(); | |
1466 | Elements.push_back(ConstantInt::get(EltTy, V2)); | |
1467 | } | |
1468 | ShadowMul = ConstantVector::get(Elements); | |
1469 | } else { | |
1470 | ConstantInt *Elt = dyn_cast<ConstantInt>(ConstArg); | |
1471 | APInt V = Elt->getValue(); | |
1472 | APInt V2 = APInt(V.getBitWidth(), 1) << V.countTrailingZeros(); | |
1473 | ShadowMul = ConstantInt::get(Elt->getType(), V2); | |
1474 | } | |
1475 | ||
1476 | IRBuilder<> IRB(&I); | |
1477 | setShadow(&I, | |
1478 | IRB.CreateMul(getShadow(OtherArg), ShadowMul, "msprop_mul_cst")); | |
1479 | setOrigin(&I, getOrigin(OtherArg)); | |
1480 | } | |
1481 | ||
1482 | void visitMul(BinaryOperator &I) { | |
1483 | Constant *constOp0 = dyn_cast<Constant>(I.getOperand(0)); | |
1484 | Constant *constOp1 = dyn_cast<Constant>(I.getOperand(1)); | |
1485 | if (constOp0 && !constOp1) | |
1486 | handleMulByConstant(I, constOp0, I.getOperand(1)); | |
1487 | else if (constOp1 && !constOp0) | |
1488 | handleMulByConstant(I, constOp1, I.getOperand(0)); | |
1489 | else | |
1490 | handleShadowOr(I); | |
1491 | } | |
1492 | ||
970d7e83 LB |
1493 | void visitFAdd(BinaryOperator &I) { handleShadowOr(I); } |
1494 | void visitFSub(BinaryOperator &I) { handleShadowOr(I); } | |
1495 | void visitFMul(BinaryOperator &I) { handleShadowOr(I); } | |
1496 | void visitAdd(BinaryOperator &I) { handleShadowOr(I); } | |
1497 | void visitSub(BinaryOperator &I) { handleShadowOr(I); } | |
1498 | void visitXor(BinaryOperator &I) { handleShadowOr(I); } | |
970d7e83 LB |
1499 | |
1500 | void handleDiv(Instruction &I) { | |
1501 | IRBuilder<> IRB(&I); | |
1502 | // Strict on the second argument. | |
1a4d82fc | 1503 | insertShadowCheck(I.getOperand(1), &I); |
970d7e83 LB |
1504 | setShadow(&I, getShadow(&I, 0)); |
1505 | setOrigin(&I, getOrigin(&I, 0)); | |
1506 | } | |
1507 | ||
1508 | void visitUDiv(BinaryOperator &I) { handleDiv(I); } | |
1509 | void visitSDiv(BinaryOperator &I) { handleDiv(I); } | |
1510 | void visitFDiv(BinaryOperator &I) { handleDiv(I); } | |
1511 | void visitURem(BinaryOperator &I) { handleDiv(I); } | |
1512 | void visitSRem(BinaryOperator &I) { handleDiv(I); } | |
1513 | void visitFRem(BinaryOperator &I) { handleDiv(I); } | |
1514 | ||
1515 | /// \brief Instrument == and != comparisons. | |
1516 | /// | |
1517 | /// Sometimes the comparison result is known even if some of the bits of the | |
1518 | /// arguments are not. | |
1519 | void handleEqualityComparison(ICmpInst &I) { | |
1520 | IRBuilder<> IRB(&I); | |
1521 | Value *A = I.getOperand(0); | |
1522 | Value *B = I.getOperand(1); | |
1523 | Value *Sa = getShadow(A); | |
1524 | Value *Sb = getShadow(B); | |
1525 | ||
1526 | // Get rid of pointers and vectors of pointers. | |
1527 | // For ints (and vectors of ints), types of A and Sa match, | |
1528 | // and this is a no-op. | |
1529 | A = IRB.CreatePointerCast(A, Sa->getType()); | |
1530 | B = IRB.CreatePointerCast(B, Sb->getType()); | |
1531 | ||
1532 | // A == B <==> (C = A^B) == 0 | |
1533 | // A != B <==> (C = A^B) != 0 | |
1534 | // Sc = Sa | Sb | |
1535 | Value *C = IRB.CreateXor(A, B); | |
1536 | Value *Sc = IRB.CreateOr(Sa, Sb); | |
1537 | // Now dealing with i = (C == 0) comparison (or C != 0, does not matter now) | |
1538 | // Result is defined if one of the following is true | |
1539 | // * there is a defined 1 bit in C | |
1540 | // * C is fully defined | |
1541 | // Si = !(C & ~Sc) && Sc | |
1542 | Value *Zero = Constant::getNullValue(Sc->getType()); | |
1543 | Value *MinusOne = Constant::getAllOnesValue(Sc->getType()); | |
1544 | Value *Si = | |
1545 | IRB.CreateAnd(IRB.CreateICmpNE(Sc, Zero), | |
1546 | IRB.CreateICmpEQ( | |
1547 | IRB.CreateAnd(IRB.CreateXor(Sc, MinusOne), C), Zero)); | |
1548 | Si->setName("_msprop_icmp"); | |
1549 | setShadow(&I, Si); | |
1550 | setOriginForNaryOp(I); | |
1551 | } | |
1552 | ||
1553 | /// \brief Build the lowest possible value of V, taking into account V's | |
1554 | /// uninitialized bits. | |
1555 | Value *getLowestPossibleValue(IRBuilder<> &IRB, Value *A, Value *Sa, | |
1556 | bool isSigned) { | |
1557 | if (isSigned) { | |
1558 | // Split shadow into sign bit and other bits. | |
1559 | Value *SaOtherBits = IRB.CreateLShr(IRB.CreateShl(Sa, 1), 1); | |
1560 | Value *SaSignBit = IRB.CreateXor(Sa, SaOtherBits); | |
1561 | // Maximise the undefined shadow bit, minimize other undefined bits. | |
1562 | return | |
1563 | IRB.CreateOr(IRB.CreateAnd(A, IRB.CreateNot(SaOtherBits)), SaSignBit); | |
1564 | } else { | |
1565 | // Minimize undefined bits. | |
1566 | return IRB.CreateAnd(A, IRB.CreateNot(Sa)); | |
1567 | } | |
1568 | } | |
1569 | ||
1570 | /// \brief Build the highest possible value of V, taking into account V's | |
1571 | /// uninitialized bits. | |
1572 | Value *getHighestPossibleValue(IRBuilder<> &IRB, Value *A, Value *Sa, | |
1573 | bool isSigned) { | |
1574 | if (isSigned) { | |
1575 | // Split shadow into sign bit and other bits. | |
1576 | Value *SaOtherBits = IRB.CreateLShr(IRB.CreateShl(Sa, 1), 1); | |
1577 | Value *SaSignBit = IRB.CreateXor(Sa, SaOtherBits); | |
1578 | // Minimise the undefined shadow bit, maximise other undefined bits. | |
1579 | return | |
1580 | IRB.CreateOr(IRB.CreateAnd(A, IRB.CreateNot(SaSignBit)), SaOtherBits); | |
1581 | } else { | |
1582 | // Maximize undefined bits. | |
1583 | return IRB.CreateOr(A, Sa); | |
1584 | } | |
1585 | } | |
1586 | ||
1587 | /// \brief Instrument relational comparisons. | |
1588 | /// | |
1589 | /// This function does exact shadow propagation for all relational | |
1590 | /// comparisons of integers, pointers and vectors of those. | |
1591 | /// FIXME: output seems suboptimal when one of the operands is a constant | |
1592 | void handleRelationalComparisonExact(ICmpInst &I) { | |
1593 | IRBuilder<> IRB(&I); | |
1594 | Value *A = I.getOperand(0); | |
1595 | Value *B = I.getOperand(1); | |
1596 | Value *Sa = getShadow(A); | |
1597 | Value *Sb = getShadow(B); | |
1598 | ||
1599 | // Get rid of pointers and vectors of pointers. | |
1600 | // For ints (and vectors of ints), types of A and Sa match, | |
1601 | // and this is a no-op. | |
1602 | A = IRB.CreatePointerCast(A, Sa->getType()); | |
1603 | B = IRB.CreatePointerCast(B, Sb->getType()); | |
1604 | ||
1605 | // Let [a0, a1] be the interval of possible values of A, taking into account | |
1606 | // its undefined bits. Let [b0, b1] be the interval of possible values of B. | |
1607 | // Then (A cmp B) is defined iff (a0 cmp b1) == (a1 cmp b0). | |
1608 | bool IsSigned = I.isSigned(); | |
1609 | Value *S1 = IRB.CreateICmp(I.getPredicate(), | |
1610 | getLowestPossibleValue(IRB, A, Sa, IsSigned), | |
1611 | getHighestPossibleValue(IRB, B, Sb, IsSigned)); | |
1612 | Value *S2 = IRB.CreateICmp(I.getPredicate(), | |
1613 | getHighestPossibleValue(IRB, A, Sa, IsSigned), | |
1614 | getLowestPossibleValue(IRB, B, Sb, IsSigned)); | |
1615 | Value *Si = IRB.CreateXor(S1, S2); | |
1616 | setShadow(&I, Si); | |
1617 | setOriginForNaryOp(I); | |
1618 | } | |
1619 | ||
1620 | /// \brief Instrument signed relational comparisons. | |
1621 | /// | |
1622 | /// Handle (x<0) and (x>=0) comparisons (essentially, sign bit tests) by | |
1623 | /// propagating the highest bit of the shadow. Everything else is delegated | |
1624 | /// to handleShadowOr(). | |
1625 | void handleSignedRelationalComparison(ICmpInst &I) { | |
1626 | Constant *constOp0 = dyn_cast<Constant>(I.getOperand(0)); | |
1627 | Constant *constOp1 = dyn_cast<Constant>(I.getOperand(1)); | |
1a4d82fc | 1628 | Value* op = nullptr; |
970d7e83 LB |
1629 | CmpInst::Predicate pre = I.getPredicate(); |
1630 | if (constOp0 && constOp0->isNullValue() && | |
1631 | (pre == CmpInst::ICMP_SGT || pre == CmpInst::ICMP_SLE)) { | |
1632 | op = I.getOperand(1); | |
1633 | } else if (constOp1 && constOp1->isNullValue() && | |
1634 | (pre == CmpInst::ICMP_SLT || pre == CmpInst::ICMP_SGE)) { | |
1635 | op = I.getOperand(0); | |
1636 | } | |
1637 | if (op) { | |
1638 | IRBuilder<> IRB(&I); | |
1639 | Value* Shadow = | |
1640 | IRB.CreateICmpSLT(getShadow(op), getCleanShadow(op), "_msprop_icmpslt"); | |
1641 | setShadow(&I, Shadow); | |
1642 | setOrigin(&I, getOrigin(op)); | |
1643 | } else { | |
1644 | handleShadowOr(I); | |
1645 | } | |
1646 | } | |
1647 | ||
1648 | void visitICmpInst(ICmpInst &I) { | |
1649 | if (!ClHandleICmp) { | |
1650 | handleShadowOr(I); | |
1651 | return; | |
1652 | } | |
1653 | if (I.isEquality()) { | |
1654 | handleEqualityComparison(I); | |
1655 | return; | |
1656 | } | |
1657 | ||
1658 | assert(I.isRelational()); | |
1659 | if (ClHandleICmpExact) { | |
1660 | handleRelationalComparisonExact(I); | |
1661 | return; | |
1662 | } | |
1663 | if (I.isSigned()) { | |
1664 | handleSignedRelationalComparison(I); | |
1665 | return; | |
1666 | } | |
1667 | ||
1668 | assert(I.isUnsigned()); | |
1669 | if ((isa<Constant>(I.getOperand(0)) || isa<Constant>(I.getOperand(1)))) { | |
1670 | handleRelationalComparisonExact(I); | |
1671 | return; | |
1672 | } | |
1673 | ||
1674 | handleShadowOr(I); | |
1675 | } | |
1676 | ||
1677 | void visitFCmpInst(FCmpInst &I) { | |
1678 | handleShadowOr(I); | |
1679 | } | |
1680 | ||
1681 | void handleShift(BinaryOperator &I) { | |
1682 | IRBuilder<> IRB(&I); | |
1683 | // If any of the S2 bits are poisoned, the whole thing is poisoned. | |
1684 | // Otherwise perform the same shift on S1. | |
1685 | Value *S1 = getShadow(&I, 0); | |
1686 | Value *S2 = getShadow(&I, 1); | |
1687 | Value *S2Conv = IRB.CreateSExt(IRB.CreateICmpNE(S2, getCleanShadow(S2)), | |
1688 | S2->getType()); | |
1689 | Value *V2 = I.getOperand(1); | |
1690 | Value *Shift = IRB.CreateBinOp(I.getOpcode(), S1, V2); | |
1691 | setShadow(&I, IRB.CreateOr(Shift, S2Conv)); | |
1692 | setOriginForNaryOp(I); | |
1693 | } | |
1694 | ||
1695 | void visitShl(BinaryOperator &I) { handleShift(I); } | |
1696 | void visitAShr(BinaryOperator &I) { handleShift(I); } | |
1697 | void visitLShr(BinaryOperator &I) { handleShift(I); } | |
1698 | ||
1699 | /// \brief Instrument llvm.memmove | |
1700 | /// | |
1701 | /// At this point we don't know if llvm.memmove will be inlined or not. | |
1702 | /// If we don't instrument it and it gets inlined, | |
1703 | /// our interceptor will not kick in and we will lose the memmove. | |
1704 | /// If we instrument the call here, but it does not get inlined, | |
1705 | /// we will memove the shadow twice: which is bad in case | |
1706 | /// of overlapping regions. So, we simply lower the intrinsic to a call. | |
1707 | /// | |
1708 | /// Similar situation exists for memcpy and memset. | |
1709 | void visitMemMoveInst(MemMoveInst &I) { | |
1710 | IRBuilder<> IRB(&I); | |
1711 | IRB.CreateCall3( | |
1712 | MS.MemmoveFn, | |
1713 | IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()), | |
1714 | IRB.CreatePointerCast(I.getArgOperand(1), IRB.getInt8PtrTy()), | |
1715 | IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)); | |
1716 | I.eraseFromParent(); | |
1717 | } | |
1718 | ||
1719 | // Similar to memmove: avoid copying shadow twice. | |
1720 | // This is somewhat unfortunate as it may slowdown small constant memcpys. | |
1721 | // FIXME: consider doing manual inline for small constant sizes and proper | |
1722 | // alignment. | |
1723 | void visitMemCpyInst(MemCpyInst &I) { | |
1724 | IRBuilder<> IRB(&I); | |
1725 | IRB.CreateCall3( | |
1726 | MS.MemcpyFn, | |
1727 | IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()), | |
1728 | IRB.CreatePointerCast(I.getArgOperand(1), IRB.getInt8PtrTy()), | |
1729 | IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)); | |
1730 | I.eraseFromParent(); | |
1731 | } | |
1732 | ||
1733 | // Same as memcpy. | |
1734 | void visitMemSetInst(MemSetInst &I) { | |
1735 | IRBuilder<> IRB(&I); | |
1736 | IRB.CreateCall3( | |
1737 | MS.MemsetFn, | |
1738 | IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()), | |
1739 | IRB.CreateIntCast(I.getArgOperand(1), IRB.getInt32Ty(), false), | |
1740 | IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)); | |
1741 | I.eraseFromParent(); | |
1742 | } | |
1743 | ||
1744 | void visitVAStartInst(VAStartInst &I) { | |
1745 | VAHelper->visitVAStartInst(I); | |
1746 | } | |
1747 | ||
1748 | void visitVACopyInst(VACopyInst &I) { | |
1749 | VAHelper->visitVACopyInst(I); | |
1750 | } | |
1751 | ||
1752 | enum IntrinsicKind { | |
1753 | IK_DoesNotAccessMemory, | |
1754 | IK_OnlyReadsMemory, | |
1755 | IK_WritesMemory | |
1756 | }; | |
1757 | ||
1758 | static IntrinsicKind getIntrinsicKind(Intrinsic::ID iid) { | |
1759 | const int DoesNotAccessMemory = IK_DoesNotAccessMemory; | |
1760 | const int OnlyReadsArgumentPointees = IK_OnlyReadsMemory; | |
1761 | const int OnlyReadsMemory = IK_OnlyReadsMemory; | |
1762 | const int OnlyAccessesArgumentPointees = IK_WritesMemory; | |
1763 | const int UnknownModRefBehavior = IK_WritesMemory; | |
1764 | #define GET_INTRINSIC_MODREF_BEHAVIOR | |
1765 | #define ModRefBehavior IntrinsicKind | |
1766 | #include "llvm/IR/Intrinsics.gen" | |
1767 | #undef ModRefBehavior | |
1768 | #undef GET_INTRINSIC_MODREF_BEHAVIOR | |
1769 | } | |
1770 | ||
1771 | /// \brief Handle vector store-like intrinsics. | |
1772 | /// | |
1773 | /// Instrument intrinsics that look like a simple SIMD store: writes memory, | |
1774 | /// has 1 pointer argument and 1 vector argument, returns void. | |
1775 | bool handleVectorStoreIntrinsic(IntrinsicInst &I) { | |
1776 | IRBuilder<> IRB(&I); | |
1777 | Value* Addr = I.getArgOperand(0); | |
1778 | Value *Shadow = getShadow(&I, 1); | |
1779 | Value *ShadowPtr = getShadowPtr(Addr, Shadow->getType(), IRB); | |
1780 | ||
1781 | // We don't know the pointer alignment (could be unaligned SSE store!). | |
1782 | // Have to assume to worst case. | |
1783 | IRB.CreateAlignedStore(Shadow, ShadowPtr, 1); | |
1784 | ||
1785 | if (ClCheckAccessAddress) | |
1a4d82fc | 1786 | insertShadowCheck(Addr, &I); |
970d7e83 LB |
1787 | |
1788 | // FIXME: use ClStoreCleanOrigin | |
1789 | // FIXME: factor out common code from materializeStores | |
1790 | if (MS.TrackOrigins) | |
85aaf69f | 1791 | IRB.CreateStore(getOrigin(&I, 1), getOriginPtr(Addr, IRB, 1)); |
970d7e83 LB |
1792 | return true; |
1793 | } | |
1794 | ||
1795 | /// \brief Handle vector load-like intrinsics. | |
1796 | /// | |
1797 | /// Instrument intrinsics that look like a simple SIMD load: reads memory, | |
1798 | /// has 1 pointer argument, returns a vector. | |
1799 | bool handleVectorLoadIntrinsic(IntrinsicInst &I) { | |
1800 | IRBuilder<> IRB(&I); | |
1801 | Value *Addr = I.getArgOperand(0); | |
1802 | ||
1803 | Type *ShadowTy = getShadowTy(&I); | |
1a4d82fc | 1804 | if (PropagateShadow) { |
970d7e83 LB |
1805 | Value *ShadowPtr = getShadowPtr(Addr, ShadowTy, IRB); |
1806 | // We don't know the pointer alignment (could be unaligned SSE load!). | |
1807 | // Have to assume to worst case. | |
1808 | setShadow(&I, IRB.CreateAlignedLoad(ShadowPtr, 1, "_msld")); | |
1809 | } else { | |
1810 | setShadow(&I, getCleanShadow(&I)); | |
1811 | } | |
1812 | ||
970d7e83 | 1813 | if (ClCheckAccessAddress) |
1a4d82fc | 1814 | insertShadowCheck(Addr, &I); |
970d7e83 LB |
1815 | |
1816 | if (MS.TrackOrigins) { | |
1a4d82fc | 1817 | if (PropagateShadow) |
85aaf69f | 1818 | setOrigin(&I, IRB.CreateLoad(getOriginPtr(Addr, IRB, 1))); |
970d7e83 LB |
1819 | else |
1820 | setOrigin(&I, getCleanOrigin()); | |
1821 | } | |
1822 | return true; | |
1823 | } | |
1824 | ||
1825 | /// \brief Handle (SIMD arithmetic)-like intrinsics. | |
1826 | /// | |
1827 | /// Instrument intrinsics with any number of arguments of the same type, | |
1828 | /// equal to the return type. The type should be simple (no aggregates or | |
1829 | /// pointers; vectors are fine). | |
1830 | /// Caller guarantees that this intrinsic does not access memory. | |
1831 | bool maybeHandleSimpleNomemIntrinsic(IntrinsicInst &I) { | |
1832 | Type *RetTy = I.getType(); | |
1833 | if (!(RetTy->isIntOrIntVectorTy() || | |
1834 | RetTy->isFPOrFPVectorTy() || | |
1835 | RetTy->isX86_MMXTy())) | |
1836 | return false; | |
1837 | ||
1838 | unsigned NumArgOperands = I.getNumArgOperands(); | |
1839 | ||
1840 | for (unsigned i = 0; i < NumArgOperands; ++i) { | |
1841 | Type *Ty = I.getArgOperand(i)->getType(); | |
1842 | if (Ty != RetTy) | |
1843 | return false; | |
1844 | } | |
1845 | ||
1846 | IRBuilder<> IRB(&I); | |
1847 | ShadowAndOriginCombiner SC(this, IRB); | |
1848 | for (unsigned i = 0; i < NumArgOperands; ++i) | |
1849 | SC.Add(I.getArgOperand(i)); | |
1850 | SC.Done(&I); | |
1851 | ||
1852 | return true; | |
1853 | } | |
1854 | ||
1855 | /// \brief Heuristically instrument unknown intrinsics. | |
1856 | /// | |
1857 | /// The main purpose of this code is to do something reasonable with all | |
1858 | /// random intrinsics we might encounter, most importantly - SIMD intrinsics. | |
1859 | /// We recognize several classes of intrinsics by their argument types and | |
1860 | /// ModRefBehaviour and apply special intrumentation when we are reasonably | |
1861 | /// sure that we know what the intrinsic does. | |
1862 | /// | |
1863 | /// We special-case intrinsics where this approach fails. See llvm.bswap | |
1864 | /// handling as an example of that. | |
1865 | bool handleUnknownIntrinsic(IntrinsicInst &I) { | |
1866 | unsigned NumArgOperands = I.getNumArgOperands(); | |
1867 | if (NumArgOperands == 0) | |
1868 | return false; | |
1869 | ||
1870 | Intrinsic::ID iid = I.getIntrinsicID(); | |
1871 | IntrinsicKind IK = getIntrinsicKind(iid); | |
1872 | bool OnlyReadsMemory = IK == IK_OnlyReadsMemory; | |
1873 | bool WritesMemory = IK == IK_WritesMemory; | |
1874 | assert(!(OnlyReadsMemory && WritesMemory)); | |
1875 | ||
1876 | if (NumArgOperands == 2 && | |
1877 | I.getArgOperand(0)->getType()->isPointerTy() && | |
1878 | I.getArgOperand(1)->getType()->isVectorTy() && | |
1879 | I.getType()->isVoidTy() && | |
1880 | WritesMemory) { | |
1881 | // This looks like a vector store. | |
1882 | return handleVectorStoreIntrinsic(I); | |
1883 | } | |
1884 | ||
1885 | if (NumArgOperands == 1 && | |
1886 | I.getArgOperand(0)->getType()->isPointerTy() && | |
1887 | I.getType()->isVectorTy() && | |
1888 | OnlyReadsMemory) { | |
1889 | // This looks like a vector load. | |
1890 | return handleVectorLoadIntrinsic(I); | |
1891 | } | |
1892 | ||
1893 | if (!OnlyReadsMemory && !WritesMemory) | |
1894 | if (maybeHandleSimpleNomemIntrinsic(I)) | |
1895 | return true; | |
1896 | ||
1897 | // FIXME: detect and handle SSE maskstore/maskload | |
1898 | return false; | |
1899 | } | |
1900 | ||
1901 | void handleBswap(IntrinsicInst &I) { | |
1902 | IRBuilder<> IRB(&I); | |
1903 | Value *Op = I.getArgOperand(0); | |
1904 | Type *OpType = Op->getType(); | |
1905 | Function *BswapFunc = Intrinsic::getDeclaration( | |
1a4d82fc | 1906 | F.getParent(), Intrinsic::bswap, makeArrayRef(&OpType, 1)); |
970d7e83 LB |
1907 | setShadow(&I, IRB.CreateCall(BswapFunc, getShadow(Op))); |
1908 | setOrigin(&I, getOrigin(Op)); | |
1909 | } | |
1910 | ||
1a4d82fc JJ |
1911 | // \brief Instrument vector convert instrinsic. |
1912 | // | |
1913 | // This function instruments intrinsics like cvtsi2ss: | |
1914 | // %Out = int_xxx_cvtyyy(%ConvertOp) | |
1915 | // or | |
1916 | // %Out = int_xxx_cvtyyy(%CopyOp, %ConvertOp) | |
1917 | // Intrinsic converts \p NumUsedElements elements of \p ConvertOp to the same | |
1918 | // number \p Out elements, and (if has 2 arguments) copies the rest of the | |
1919 | // elements from \p CopyOp. | |
1920 | // In most cases conversion involves floating-point value which may trigger a | |
1921 | // hardware exception when not fully initialized. For this reason we require | |
1922 | // \p ConvertOp[0:NumUsedElements] to be fully initialized and trap otherwise. | |
1923 | // We copy the shadow of \p CopyOp[NumUsedElements:] to \p | |
1924 | // Out[NumUsedElements:]. This means that intrinsics without \p CopyOp always | |
1925 | // return a fully initialized value. | |
1926 | void handleVectorConvertIntrinsic(IntrinsicInst &I, int NumUsedElements) { | |
1927 | IRBuilder<> IRB(&I); | |
1928 | Value *CopyOp, *ConvertOp; | |
1929 | ||
1930 | switch (I.getNumArgOperands()) { | |
1931 | case 2: | |
1932 | CopyOp = I.getArgOperand(0); | |
1933 | ConvertOp = I.getArgOperand(1); | |
1934 | break; | |
1935 | case 1: | |
1936 | ConvertOp = I.getArgOperand(0); | |
1937 | CopyOp = nullptr; | |
1938 | break; | |
1939 | default: | |
1940 | llvm_unreachable("Cvt intrinsic with unsupported number of arguments."); | |
1941 | } | |
1942 | ||
1943 | // The first *NumUsedElements* elements of ConvertOp are converted to the | |
1944 | // same number of output elements. The rest of the output is copied from | |
1945 | // CopyOp, or (if not available) filled with zeroes. | |
1946 | // Combine shadow for elements of ConvertOp that are used in this operation, | |
1947 | // and insert a check. | |
1948 | // FIXME: consider propagating shadow of ConvertOp, at least in the case of | |
1949 | // int->any conversion. | |
1950 | Value *ConvertShadow = getShadow(ConvertOp); | |
1951 | Value *AggShadow = nullptr; | |
1952 | if (ConvertOp->getType()->isVectorTy()) { | |
1953 | AggShadow = IRB.CreateExtractElement( | |
1954 | ConvertShadow, ConstantInt::get(IRB.getInt32Ty(), 0)); | |
1955 | for (int i = 1; i < NumUsedElements; ++i) { | |
1956 | Value *MoreShadow = IRB.CreateExtractElement( | |
1957 | ConvertShadow, ConstantInt::get(IRB.getInt32Ty(), i)); | |
1958 | AggShadow = IRB.CreateOr(AggShadow, MoreShadow); | |
1959 | } | |
1960 | } else { | |
1961 | AggShadow = ConvertShadow; | |
1962 | } | |
1963 | assert(AggShadow->getType()->isIntegerTy()); | |
1964 | insertShadowCheck(AggShadow, getOrigin(ConvertOp), &I); | |
1965 | ||
1966 | // Build result shadow by zero-filling parts of CopyOp shadow that come from | |
1967 | // ConvertOp. | |
1968 | if (CopyOp) { | |
1969 | assert(CopyOp->getType() == I.getType()); | |
1970 | assert(CopyOp->getType()->isVectorTy()); | |
1971 | Value *ResultShadow = getShadow(CopyOp); | |
1972 | Type *EltTy = ResultShadow->getType()->getVectorElementType(); | |
1973 | for (int i = 0; i < NumUsedElements; ++i) { | |
1974 | ResultShadow = IRB.CreateInsertElement( | |
1975 | ResultShadow, ConstantInt::getNullValue(EltTy), | |
1976 | ConstantInt::get(IRB.getInt32Ty(), i)); | |
1977 | } | |
1978 | setShadow(&I, ResultShadow); | |
1979 | setOrigin(&I, getOrigin(CopyOp)); | |
1980 | } else { | |
1981 | setShadow(&I, getCleanShadow(&I)); | |
85aaf69f | 1982 | setOrigin(&I, getCleanOrigin()); |
1a4d82fc JJ |
1983 | } |
1984 | } | |
1985 | ||
1986 | // Given a scalar or vector, extract lower 64 bits (or less), and return all | |
1987 | // zeroes if it is zero, and all ones otherwise. | |
1988 | Value *Lower64ShadowExtend(IRBuilder<> &IRB, Value *S, Type *T) { | |
1989 | if (S->getType()->isVectorTy()) | |
1990 | S = CreateShadowCast(IRB, S, IRB.getInt64Ty(), /* Signed */ true); | |
1991 | assert(S->getType()->getPrimitiveSizeInBits() <= 64); | |
1992 | Value *S2 = IRB.CreateICmpNE(S, getCleanShadow(S)); | |
1993 | return CreateShadowCast(IRB, S2, T, /* Signed */ true); | |
1994 | } | |
1995 | ||
1996 | Value *VariableShadowExtend(IRBuilder<> &IRB, Value *S) { | |
1997 | Type *T = S->getType(); | |
1998 | assert(T->isVectorTy()); | |
1999 | Value *S2 = IRB.CreateICmpNE(S, getCleanShadow(S)); | |
2000 | return IRB.CreateSExt(S2, T); | |
2001 | } | |
2002 | ||
2003 | // \brief Instrument vector shift instrinsic. | |
2004 | // | |
2005 | // This function instruments intrinsics like int_x86_avx2_psll_w. | |
2006 | // Intrinsic shifts %In by %ShiftSize bits. | |
2007 | // %ShiftSize may be a vector. In that case the lower 64 bits determine shift | |
2008 | // size, and the rest is ignored. Behavior is defined even if shift size is | |
2009 | // greater than register (or field) width. | |
2010 | void handleVectorShiftIntrinsic(IntrinsicInst &I, bool Variable) { | |
2011 | assert(I.getNumArgOperands() == 2); | |
2012 | IRBuilder<> IRB(&I); | |
2013 | // If any of the S2 bits are poisoned, the whole thing is poisoned. | |
2014 | // Otherwise perform the same shift on S1. | |
2015 | Value *S1 = getShadow(&I, 0); | |
2016 | Value *S2 = getShadow(&I, 1); | |
2017 | Value *S2Conv = Variable ? VariableShadowExtend(IRB, S2) | |
2018 | : Lower64ShadowExtend(IRB, S2, getShadowTy(&I)); | |
2019 | Value *V1 = I.getOperand(0); | |
2020 | Value *V2 = I.getOperand(1); | |
2021 | Value *Shift = IRB.CreateCall2(I.getCalledValue(), | |
2022 | IRB.CreateBitCast(S1, V1->getType()), V2); | |
2023 | Shift = IRB.CreateBitCast(Shift, getShadowTy(&I)); | |
2024 | setShadow(&I, IRB.CreateOr(Shift, S2Conv)); | |
2025 | setOriginForNaryOp(I); | |
2026 | } | |
2027 | ||
2028 | // \brief Get an X86_MMX-sized vector type. | |
2029 | Type *getMMXVectorTy(unsigned EltSizeInBits) { | |
2030 | const unsigned X86_MMXSizeInBits = 64; | |
2031 | return VectorType::get(IntegerType::get(*MS.C, EltSizeInBits), | |
2032 | X86_MMXSizeInBits / EltSizeInBits); | |
2033 | } | |
2034 | ||
2035 | // \brief Returns a signed counterpart for an (un)signed-saturate-and-pack | |
2036 | // intrinsic. | |
2037 | Intrinsic::ID getSignedPackIntrinsic(Intrinsic::ID id) { | |
2038 | switch (id) { | |
2039 | case llvm::Intrinsic::x86_sse2_packsswb_128: | |
2040 | case llvm::Intrinsic::x86_sse2_packuswb_128: | |
2041 | return llvm::Intrinsic::x86_sse2_packsswb_128; | |
2042 | ||
2043 | case llvm::Intrinsic::x86_sse2_packssdw_128: | |
2044 | case llvm::Intrinsic::x86_sse41_packusdw: | |
2045 | return llvm::Intrinsic::x86_sse2_packssdw_128; | |
2046 | ||
2047 | case llvm::Intrinsic::x86_avx2_packsswb: | |
2048 | case llvm::Intrinsic::x86_avx2_packuswb: | |
2049 | return llvm::Intrinsic::x86_avx2_packsswb; | |
2050 | ||
2051 | case llvm::Intrinsic::x86_avx2_packssdw: | |
2052 | case llvm::Intrinsic::x86_avx2_packusdw: | |
2053 | return llvm::Intrinsic::x86_avx2_packssdw; | |
2054 | ||
2055 | case llvm::Intrinsic::x86_mmx_packsswb: | |
2056 | case llvm::Intrinsic::x86_mmx_packuswb: | |
2057 | return llvm::Intrinsic::x86_mmx_packsswb; | |
2058 | ||
2059 | case llvm::Intrinsic::x86_mmx_packssdw: | |
2060 | return llvm::Intrinsic::x86_mmx_packssdw; | |
2061 | default: | |
2062 | llvm_unreachable("unexpected intrinsic id"); | |
2063 | } | |
2064 | } | |
2065 | ||
2066 | // \brief Instrument vector pack instrinsic. | |
2067 | // | |
2068 | // This function instruments intrinsics like x86_mmx_packsswb, that | |
2069 | // packs elements of 2 input vectors into half as many bits with saturation. | |
2070 | // Shadow is propagated with the signed variant of the same intrinsic applied | |
2071 | // to sext(Sa != zeroinitializer), sext(Sb != zeroinitializer). | |
2072 | // EltSizeInBits is used only for x86mmx arguments. | |
2073 | void handleVectorPackIntrinsic(IntrinsicInst &I, unsigned EltSizeInBits = 0) { | |
2074 | assert(I.getNumArgOperands() == 2); | |
2075 | bool isX86_MMX = I.getOperand(0)->getType()->isX86_MMXTy(); | |
2076 | IRBuilder<> IRB(&I); | |
2077 | Value *S1 = getShadow(&I, 0); | |
2078 | Value *S2 = getShadow(&I, 1); | |
2079 | assert(isX86_MMX || S1->getType()->isVectorTy()); | |
2080 | ||
2081 | // SExt and ICmpNE below must apply to individual elements of input vectors. | |
2082 | // In case of x86mmx arguments, cast them to appropriate vector types and | |
2083 | // back. | |
2084 | Type *T = isX86_MMX ? getMMXVectorTy(EltSizeInBits) : S1->getType(); | |
2085 | if (isX86_MMX) { | |
2086 | S1 = IRB.CreateBitCast(S1, T); | |
2087 | S2 = IRB.CreateBitCast(S2, T); | |
2088 | } | |
2089 | Value *S1_ext = IRB.CreateSExt( | |
2090 | IRB.CreateICmpNE(S1, llvm::Constant::getNullValue(T)), T); | |
2091 | Value *S2_ext = IRB.CreateSExt( | |
2092 | IRB.CreateICmpNE(S2, llvm::Constant::getNullValue(T)), T); | |
2093 | if (isX86_MMX) { | |
2094 | Type *X86_MMXTy = Type::getX86_MMXTy(*MS.C); | |
2095 | S1_ext = IRB.CreateBitCast(S1_ext, X86_MMXTy); | |
2096 | S2_ext = IRB.CreateBitCast(S2_ext, X86_MMXTy); | |
2097 | } | |
2098 | ||
2099 | Function *ShadowFn = Intrinsic::getDeclaration( | |
2100 | F.getParent(), getSignedPackIntrinsic(I.getIntrinsicID())); | |
2101 | ||
2102 | Value *S = IRB.CreateCall2(ShadowFn, S1_ext, S2_ext, "_msprop_vector_pack"); | |
2103 | if (isX86_MMX) S = IRB.CreateBitCast(S, getShadowTy(&I)); | |
2104 | setShadow(&I, S); | |
2105 | setOriginForNaryOp(I); | |
2106 | } | |
2107 | ||
2108 | // \brief Instrument sum-of-absolute-differencies intrinsic. | |
2109 | void handleVectorSadIntrinsic(IntrinsicInst &I) { | |
2110 | const unsigned SignificantBitsPerResultElement = 16; | |
2111 | bool isX86_MMX = I.getOperand(0)->getType()->isX86_MMXTy(); | |
2112 | Type *ResTy = isX86_MMX ? IntegerType::get(*MS.C, 64) : I.getType(); | |
2113 | unsigned ZeroBitsPerResultElement = | |
2114 | ResTy->getScalarSizeInBits() - SignificantBitsPerResultElement; | |
2115 | ||
2116 | IRBuilder<> IRB(&I); | |
2117 | Value *S = IRB.CreateOr(getShadow(&I, 0), getShadow(&I, 1)); | |
2118 | S = IRB.CreateBitCast(S, ResTy); | |
2119 | S = IRB.CreateSExt(IRB.CreateICmpNE(S, Constant::getNullValue(ResTy)), | |
2120 | ResTy); | |
2121 | S = IRB.CreateLShr(S, ZeroBitsPerResultElement); | |
2122 | S = IRB.CreateBitCast(S, getShadowTy(&I)); | |
2123 | setShadow(&I, S); | |
2124 | setOriginForNaryOp(I); | |
2125 | } | |
2126 | ||
2127 | // \brief Instrument multiply-add intrinsic. | |
2128 | void handleVectorPmaddIntrinsic(IntrinsicInst &I, | |
2129 | unsigned EltSizeInBits = 0) { | |
2130 | bool isX86_MMX = I.getOperand(0)->getType()->isX86_MMXTy(); | |
2131 | Type *ResTy = isX86_MMX ? getMMXVectorTy(EltSizeInBits * 2) : I.getType(); | |
2132 | IRBuilder<> IRB(&I); | |
2133 | Value *S = IRB.CreateOr(getShadow(&I, 0), getShadow(&I, 1)); | |
2134 | S = IRB.CreateBitCast(S, ResTy); | |
2135 | S = IRB.CreateSExt(IRB.CreateICmpNE(S, Constant::getNullValue(ResTy)), | |
2136 | ResTy); | |
2137 | S = IRB.CreateBitCast(S, getShadowTy(&I)); | |
2138 | setShadow(&I, S); | |
2139 | setOriginForNaryOp(I); | |
2140 | } | |
2141 | ||
970d7e83 LB |
2142 | void visitIntrinsicInst(IntrinsicInst &I) { |
2143 | switch (I.getIntrinsicID()) { | |
2144 | case llvm::Intrinsic::bswap: | |
2145 | handleBswap(I); | |
2146 | break; | |
1a4d82fc JJ |
2147 | case llvm::Intrinsic::x86_avx512_cvtsd2usi64: |
2148 | case llvm::Intrinsic::x86_avx512_cvtsd2usi: | |
2149 | case llvm::Intrinsic::x86_avx512_cvtss2usi64: | |
2150 | case llvm::Intrinsic::x86_avx512_cvtss2usi: | |
2151 | case llvm::Intrinsic::x86_avx512_cvttss2usi64: | |
2152 | case llvm::Intrinsic::x86_avx512_cvttss2usi: | |
2153 | case llvm::Intrinsic::x86_avx512_cvttsd2usi64: | |
2154 | case llvm::Intrinsic::x86_avx512_cvttsd2usi: | |
2155 | case llvm::Intrinsic::x86_avx512_cvtusi2sd: | |
2156 | case llvm::Intrinsic::x86_avx512_cvtusi2ss: | |
2157 | case llvm::Intrinsic::x86_avx512_cvtusi642sd: | |
2158 | case llvm::Intrinsic::x86_avx512_cvtusi642ss: | |
2159 | case llvm::Intrinsic::x86_sse2_cvtsd2si64: | |
2160 | case llvm::Intrinsic::x86_sse2_cvtsd2si: | |
2161 | case llvm::Intrinsic::x86_sse2_cvtsd2ss: | |
2162 | case llvm::Intrinsic::x86_sse2_cvtsi2sd: | |
2163 | case llvm::Intrinsic::x86_sse2_cvtsi642sd: | |
2164 | case llvm::Intrinsic::x86_sse2_cvtss2sd: | |
2165 | case llvm::Intrinsic::x86_sse2_cvttsd2si64: | |
2166 | case llvm::Intrinsic::x86_sse2_cvttsd2si: | |
2167 | case llvm::Intrinsic::x86_sse_cvtsi2ss: | |
2168 | case llvm::Intrinsic::x86_sse_cvtsi642ss: | |
2169 | case llvm::Intrinsic::x86_sse_cvtss2si64: | |
2170 | case llvm::Intrinsic::x86_sse_cvtss2si: | |
2171 | case llvm::Intrinsic::x86_sse_cvttss2si64: | |
2172 | case llvm::Intrinsic::x86_sse_cvttss2si: | |
2173 | handleVectorConvertIntrinsic(I, 1); | |
2174 | break; | |
2175 | case llvm::Intrinsic::x86_sse2_cvtdq2pd: | |
2176 | case llvm::Intrinsic::x86_sse2_cvtps2pd: | |
2177 | case llvm::Intrinsic::x86_sse_cvtps2pi: | |
2178 | case llvm::Intrinsic::x86_sse_cvttps2pi: | |
2179 | handleVectorConvertIntrinsic(I, 2); | |
2180 | break; | |
2181 | case llvm::Intrinsic::x86_avx512_psll_dq: | |
2182 | case llvm::Intrinsic::x86_avx512_psrl_dq: | |
2183 | case llvm::Intrinsic::x86_avx2_psll_w: | |
2184 | case llvm::Intrinsic::x86_avx2_psll_d: | |
2185 | case llvm::Intrinsic::x86_avx2_psll_q: | |
2186 | case llvm::Intrinsic::x86_avx2_pslli_w: | |
2187 | case llvm::Intrinsic::x86_avx2_pslli_d: | |
2188 | case llvm::Intrinsic::x86_avx2_pslli_q: | |
2189 | case llvm::Intrinsic::x86_avx2_psll_dq: | |
2190 | case llvm::Intrinsic::x86_avx2_psrl_w: | |
2191 | case llvm::Intrinsic::x86_avx2_psrl_d: | |
2192 | case llvm::Intrinsic::x86_avx2_psrl_q: | |
2193 | case llvm::Intrinsic::x86_avx2_psra_w: | |
2194 | case llvm::Intrinsic::x86_avx2_psra_d: | |
2195 | case llvm::Intrinsic::x86_avx2_psrli_w: | |
2196 | case llvm::Intrinsic::x86_avx2_psrli_d: | |
2197 | case llvm::Intrinsic::x86_avx2_psrli_q: | |
2198 | case llvm::Intrinsic::x86_avx2_psrai_w: | |
2199 | case llvm::Intrinsic::x86_avx2_psrai_d: | |
2200 | case llvm::Intrinsic::x86_avx2_psrl_dq: | |
2201 | case llvm::Intrinsic::x86_sse2_psll_w: | |
2202 | case llvm::Intrinsic::x86_sse2_psll_d: | |
2203 | case llvm::Intrinsic::x86_sse2_psll_q: | |
2204 | case llvm::Intrinsic::x86_sse2_pslli_w: | |
2205 | case llvm::Intrinsic::x86_sse2_pslli_d: | |
2206 | case llvm::Intrinsic::x86_sse2_pslli_q: | |
2207 | case llvm::Intrinsic::x86_sse2_psll_dq: | |
2208 | case llvm::Intrinsic::x86_sse2_psrl_w: | |
2209 | case llvm::Intrinsic::x86_sse2_psrl_d: | |
2210 | case llvm::Intrinsic::x86_sse2_psrl_q: | |
2211 | case llvm::Intrinsic::x86_sse2_psra_w: | |
2212 | case llvm::Intrinsic::x86_sse2_psra_d: | |
2213 | case llvm::Intrinsic::x86_sse2_psrli_w: | |
2214 | case llvm::Intrinsic::x86_sse2_psrli_d: | |
2215 | case llvm::Intrinsic::x86_sse2_psrli_q: | |
2216 | case llvm::Intrinsic::x86_sse2_psrai_w: | |
2217 | case llvm::Intrinsic::x86_sse2_psrai_d: | |
2218 | case llvm::Intrinsic::x86_sse2_psrl_dq: | |
2219 | case llvm::Intrinsic::x86_mmx_psll_w: | |
2220 | case llvm::Intrinsic::x86_mmx_psll_d: | |
2221 | case llvm::Intrinsic::x86_mmx_psll_q: | |
2222 | case llvm::Intrinsic::x86_mmx_pslli_w: | |
2223 | case llvm::Intrinsic::x86_mmx_pslli_d: | |
2224 | case llvm::Intrinsic::x86_mmx_pslli_q: | |
2225 | case llvm::Intrinsic::x86_mmx_psrl_w: | |
2226 | case llvm::Intrinsic::x86_mmx_psrl_d: | |
2227 | case llvm::Intrinsic::x86_mmx_psrl_q: | |
2228 | case llvm::Intrinsic::x86_mmx_psra_w: | |
2229 | case llvm::Intrinsic::x86_mmx_psra_d: | |
2230 | case llvm::Intrinsic::x86_mmx_psrli_w: | |
2231 | case llvm::Intrinsic::x86_mmx_psrli_d: | |
2232 | case llvm::Intrinsic::x86_mmx_psrli_q: | |
2233 | case llvm::Intrinsic::x86_mmx_psrai_w: | |
2234 | case llvm::Intrinsic::x86_mmx_psrai_d: | |
2235 | handleVectorShiftIntrinsic(I, /* Variable */ false); | |
2236 | break; | |
2237 | case llvm::Intrinsic::x86_avx2_psllv_d: | |
2238 | case llvm::Intrinsic::x86_avx2_psllv_d_256: | |
2239 | case llvm::Intrinsic::x86_avx2_psllv_q: | |
2240 | case llvm::Intrinsic::x86_avx2_psllv_q_256: | |
2241 | case llvm::Intrinsic::x86_avx2_psrlv_d: | |
2242 | case llvm::Intrinsic::x86_avx2_psrlv_d_256: | |
2243 | case llvm::Intrinsic::x86_avx2_psrlv_q: | |
2244 | case llvm::Intrinsic::x86_avx2_psrlv_q_256: | |
2245 | case llvm::Intrinsic::x86_avx2_psrav_d: | |
2246 | case llvm::Intrinsic::x86_avx2_psrav_d_256: | |
2247 | handleVectorShiftIntrinsic(I, /* Variable */ true); | |
2248 | break; | |
2249 | ||
2250 | // Byte shifts are not implemented. | |
2251 | // case llvm::Intrinsic::x86_avx512_psll_dq_bs: | |
2252 | // case llvm::Intrinsic::x86_avx512_psrl_dq_bs: | |
2253 | // case llvm::Intrinsic::x86_avx2_psll_dq_bs: | |
2254 | // case llvm::Intrinsic::x86_avx2_psrl_dq_bs: | |
2255 | // case llvm::Intrinsic::x86_sse2_psll_dq_bs: | |
2256 | // case llvm::Intrinsic::x86_sse2_psrl_dq_bs: | |
2257 | ||
2258 | case llvm::Intrinsic::x86_sse2_packsswb_128: | |
2259 | case llvm::Intrinsic::x86_sse2_packssdw_128: | |
2260 | case llvm::Intrinsic::x86_sse2_packuswb_128: | |
2261 | case llvm::Intrinsic::x86_sse41_packusdw: | |
2262 | case llvm::Intrinsic::x86_avx2_packsswb: | |
2263 | case llvm::Intrinsic::x86_avx2_packssdw: | |
2264 | case llvm::Intrinsic::x86_avx2_packuswb: | |
2265 | case llvm::Intrinsic::x86_avx2_packusdw: | |
2266 | handleVectorPackIntrinsic(I); | |
2267 | break; | |
2268 | ||
2269 | case llvm::Intrinsic::x86_mmx_packsswb: | |
2270 | case llvm::Intrinsic::x86_mmx_packuswb: | |
2271 | handleVectorPackIntrinsic(I, 16); | |
2272 | break; | |
2273 | ||
2274 | case llvm::Intrinsic::x86_mmx_packssdw: | |
2275 | handleVectorPackIntrinsic(I, 32); | |
2276 | break; | |
2277 | ||
2278 | case llvm::Intrinsic::x86_mmx_psad_bw: | |
2279 | case llvm::Intrinsic::x86_sse2_psad_bw: | |
2280 | case llvm::Intrinsic::x86_avx2_psad_bw: | |
2281 | handleVectorSadIntrinsic(I); | |
2282 | break; | |
2283 | ||
2284 | case llvm::Intrinsic::x86_sse2_pmadd_wd: | |
2285 | case llvm::Intrinsic::x86_avx2_pmadd_wd: | |
2286 | case llvm::Intrinsic::x86_ssse3_pmadd_ub_sw_128: | |
2287 | case llvm::Intrinsic::x86_avx2_pmadd_ub_sw: | |
2288 | handleVectorPmaddIntrinsic(I); | |
2289 | break; | |
2290 | ||
2291 | case llvm::Intrinsic::x86_ssse3_pmadd_ub_sw: | |
2292 | handleVectorPmaddIntrinsic(I, 8); | |
2293 | break; | |
2294 | ||
2295 | case llvm::Intrinsic::x86_mmx_pmadd_wd: | |
2296 | handleVectorPmaddIntrinsic(I, 16); | |
2297 | break; | |
2298 | ||
970d7e83 LB |
2299 | default: |
2300 | if (!handleUnknownIntrinsic(I)) | |
2301 | visitInstruction(I); | |
2302 | break; | |
2303 | } | |
2304 | } | |
2305 | ||
2306 | void visitCallSite(CallSite CS) { | |
2307 | Instruction &I = *CS.getInstruction(); | |
2308 | assert((CS.isCall() || CS.isInvoke()) && "Unknown type of CallSite"); | |
2309 | if (CS.isCall()) { | |
2310 | CallInst *Call = cast<CallInst>(&I); | |
2311 | ||
2312 | // For inline asm, do the usual thing: check argument shadow and mark all | |
2313 | // outputs as clean. Note that any side effects of the inline asm that are | |
2314 | // not immediately visible in its constraints are not handled. | |
2315 | if (Call->isInlineAsm()) { | |
2316 | visitInstruction(I); | |
2317 | return; | |
2318 | } | |
2319 | ||
970d7e83 LB |
2320 | assert(!isa<IntrinsicInst>(&I) && "intrinsics are handled elsewhere"); |
2321 | ||
2322 | // We are going to insert code that relies on the fact that the callee | |
2323 | // will become a non-readonly function after it is instrumented by us. To | |
2324 | // prevent this code from being optimized out, mark that function | |
2325 | // non-readonly in advance. | |
2326 | if (Function *Func = Call->getCalledFunction()) { | |
2327 | // Clear out readonly/readnone attributes. | |
2328 | AttrBuilder B; | |
2329 | B.addAttribute(Attribute::ReadOnly) | |
2330 | .addAttribute(Attribute::ReadNone); | |
2331 | Func->removeAttributes(AttributeSet::FunctionIndex, | |
2332 | AttributeSet::get(Func->getContext(), | |
2333 | AttributeSet::FunctionIndex, | |
2334 | B)); | |
2335 | } | |
2336 | } | |
2337 | IRBuilder<> IRB(&I); | |
1a4d82fc | 2338 | |
970d7e83 LB |
2339 | unsigned ArgOffset = 0; |
2340 | DEBUG(dbgs() << " CallSite: " << I << "\n"); | |
2341 | for (CallSite::arg_iterator ArgIt = CS.arg_begin(), End = CS.arg_end(); | |
2342 | ArgIt != End; ++ArgIt) { | |
2343 | Value *A = *ArgIt; | |
2344 | unsigned i = ArgIt - CS.arg_begin(); | |
2345 | if (!A->getType()->isSized()) { | |
2346 | DEBUG(dbgs() << "Arg " << i << " is not sized: " << I << "\n"); | |
2347 | continue; | |
2348 | } | |
2349 | unsigned Size = 0; | |
1a4d82fc | 2350 | Value *Store = nullptr; |
970d7e83 LB |
2351 | // Compute the Shadow for arg even if it is ByVal, because |
2352 | // in that case getShadow() will copy the actual arg shadow to | |
2353 | // __msan_param_tls. | |
2354 | Value *ArgShadow = getShadow(A); | |
2355 | Value *ArgShadowBase = getShadowPtrForArgument(A, IRB, ArgOffset); | |
2356 | DEBUG(dbgs() << " Arg#" << i << ": " << *A << | |
2357 | " Shadow: " << *ArgShadow << "\n"); | |
1a4d82fc | 2358 | bool ArgIsInitialized = false; |
970d7e83 LB |
2359 | if (CS.paramHasAttr(i + 1, Attribute::ByVal)) { |
2360 | assert(A->getType()->isPointerTy() && | |
2361 | "ByVal argument is not a pointer!"); | |
1a4d82fc | 2362 | Size = MS.DL->getTypeAllocSize(A->getType()->getPointerElementType()); |
85aaf69f SL |
2363 | if (ArgOffset + Size > kParamTLSSize) break; |
2364 | unsigned ParamAlignment = CS.getParamAlignment(i + 1); | |
2365 | unsigned Alignment = std::min(ParamAlignment, kShadowTLSAlignment); | |
970d7e83 LB |
2366 | Store = IRB.CreateMemCpy(ArgShadowBase, |
2367 | getShadowPtr(A, Type::getInt8Ty(*MS.C), IRB), | |
2368 | Size, Alignment); | |
2369 | } else { | |
1a4d82fc | 2370 | Size = MS.DL->getTypeAllocSize(A->getType()); |
85aaf69f | 2371 | if (ArgOffset + Size > kParamTLSSize) break; |
970d7e83 LB |
2372 | Store = IRB.CreateAlignedStore(ArgShadow, ArgShadowBase, |
2373 | kShadowTLSAlignment); | |
1a4d82fc JJ |
2374 | Constant *Cst = dyn_cast<Constant>(ArgShadow); |
2375 | if (Cst && Cst->isNullValue()) ArgIsInitialized = true; | |
970d7e83 | 2376 | } |
1a4d82fc | 2377 | if (MS.TrackOrigins && !ArgIsInitialized) |
970d7e83 LB |
2378 | IRB.CreateStore(getOrigin(A), |
2379 | getOriginPtrForArgument(A, IRB, ArgOffset)); | |
2380 | (void)Store; | |
1a4d82fc | 2381 | assert(Size != 0 && Store != nullptr); |
970d7e83 | 2382 | DEBUG(dbgs() << " Param:" << *Store << "\n"); |
85aaf69f | 2383 | ArgOffset += RoundUpToAlignment(Size, 8); |
970d7e83 LB |
2384 | } |
2385 | DEBUG(dbgs() << " done with call args\n"); | |
2386 | ||
2387 | FunctionType *FT = | |
1a4d82fc | 2388 | cast<FunctionType>(CS.getCalledValue()->getType()->getContainedType(0)); |
970d7e83 LB |
2389 | if (FT->isVarArg()) { |
2390 | VAHelper->visitCallSite(CS, IRB); | |
2391 | } | |
2392 | ||
2393 | // Now, get the shadow for the RetVal. | |
2394 | if (!I.getType()->isSized()) return; | |
2395 | IRBuilder<> IRBBefore(&I); | |
1a4d82fc | 2396 | // Until we have full dynamic coverage, make sure the retval shadow is 0. |
970d7e83 LB |
2397 | Value *Base = getShadowPtrForRetval(&I, IRBBefore); |
2398 | IRBBefore.CreateAlignedStore(getCleanShadow(&I), Base, kShadowTLSAlignment); | |
1a4d82fc | 2399 | Instruction *NextInsn = nullptr; |
970d7e83 LB |
2400 | if (CS.isCall()) { |
2401 | NextInsn = I.getNextNode(); | |
2402 | } else { | |
2403 | BasicBlock *NormalDest = cast<InvokeInst>(&I)->getNormalDest(); | |
2404 | if (!NormalDest->getSinglePredecessor()) { | |
2405 | // FIXME: this case is tricky, so we are just conservative here. | |
2406 | // Perhaps we need to split the edge between this BB and NormalDest, | |
2407 | // but a naive attempt to use SplitEdge leads to a crash. | |
2408 | setShadow(&I, getCleanShadow(&I)); | |
2409 | setOrigin(&I, getCleanOrigin()); | |
2410 | return; | |
2411 | } | |
2412 | NextInsn = NormalDest->getFirstInsertionPt(); | |
2413 | assert(NextInsn && | |
2414 | "Could not find insertion point for retval shadow load"); | |
2415 | } | |
2416 | IRBuilder<> IRBAfter(NextInsn); | |
2417 | Value *RetvalShadow = | |
2418 | IRBAfter.CreateAlignedLoad(getShadowPtrForRetval(&I, IRBAfter), | |
2419 | kShadowTLSAlignment, "_msret"); | |
2420 | setShadow(&I, RetvalShadow); | |
2421 | if (MS.TrackOrigins) | |
2422 | setOrigin(&I, IRBAfter.CreateLoad(getOriginPtrForRetval(IRBAfter))); | |
2423 | } | |
2424 | ||
2425 | void visitReturnInst(ReturnInst &I) { | |
2426 | IRBuilder<> IRB(&I); | |
1a4d82fc JJ |
2427 | Value *RetVal = I.getReturnValue(); |
2428 | if (!RetVal) return; | |
2429 | Value *ShadowPtr = getShadowPtrForRetval(RetVal, IRB); | |
2430 | if (CheckReturnValue) { | |
2431 | insertShadowCheck(RetVal, &I); | |
2432 | Value *Shadow = getCleanShadow(RetVal); | |
2433 | IRB.CreateAlignedStore(Shadow, ShadowPtr, kShadowTLSAlignment); | |
2434 | } else { | |
970d7e83 | 2435 | Value *Shadow = getShadow(RetVal); |
970d7e83 | 2436 | IRB.CreateAlignedStore(Shadow, ShadowPtr, kShadowTLSAlignment); |
1a4d82fc | 2437 | // FIXME: make it conditional if ClStoreCleanOrigin==0 |
970d7e83 LB |
2438 | if (MS.TrackOrigins) |
2439 | IRB.CreateStore(getOrigin(RetVal), getOriginPtrForRetval(IRB)); | |
2440 | } | |
2441 | } | |
2442 | ||
2443 | void visitPHINode(PHINode &I) { | |
2444 | IRBuilder<> IRB(&I); | |
1a4d82fc JJ |
2445 | if (!PropagateShadow) { |
2446 | setShadow(&I, getCleanShadow(&I)); | |
85aaf69f | 2447 | setOrigin(&I, getCleanOrigin()); |
1a4d82fc JJ |
2448 | return; |
2449 | } | |
2450 | ||
970d7e83 LB |
2451 | ShadowPHINodes.push_back(&I); |
2452 | setShadow(&I, IRB.CreatePHI(getShadowTy(&I), I.getNumIncomingValues(), | |
2453 | "_msphi_s")); | |
2454 | if (MS.TrackOrigins) | |
2455 | setOrigin(&I, IRB.CreatePHI(MS.OriginTy, I.getNumIncomingValues(), | |
2456 | "_msphi_o")); | |
2457 | } | |
2458 | ||
2459 | void visitAllocaInst(AllocaInst &I) { | |
2460 | setShadow(&I, getCleanShadow(&I)); | |
85aaf69f | 2461 | setOrigin(&I, getCleanOrigin()); |
970d7e83 | 2462 | IRBuilder<> IRB(I.getNextNode()); |
1a4d82fc JJ |
2463 | uint64_t Size = MS.DL->getTypeAllocSize(I.getAllocatedType()); |
2464 | if (PoisonStack && ClPoisonStackWithCall) { | |
970d7e83 LB |
2465 | IRB.CreateCall2(MS.MsanPoisonStackFn, |
2466 | IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), | |
2467 | ConstantInt::get(MS.IntptrTy, Size)); | |
2468 | } else { | |
2469 | Value *ShadowBase = getShadowPtr(&I, Type::getInt8PtrTy(*MS.C), IRB); | |
1a4d82fc JJ |
2470 | Value *PoisonValue = IRB.getInt8(PoisonStack ? ClPoisonStackPattern : 0); |
2471 | IRB.CreateMemSet(ShadowBase, PoisonValue, Size, I.getAlignment()); | |
970d7e83 LB |
2472 | } |
2473 | ||
1a4d82fc | 2474 | if (PoisonStack && MS.TrackOrigins) { |
970d7e83 LB |
2475 | SmallString<2048> StackDescriptionStorage; |
2476 | raw_svector_ostream StackDescription(StackDescriptionStorage); | |
2477 | // We create a string with a description of the stack allocation and | |
2478 | // pass it into __msan_set_alloca_origin. | |
2479 | // It will be printed by the run-time if stack-originated UMR is found. | |
2480 | // The first 4 bytes of the string are set to '----' and will be replaced | |
2481 | // by __msan_va_arg_overflow_size_tls at the first call. | |
2482 | StackDescription << "----" << I.getName() << "@" << F.getName(); | |
2483 | Value *Descr = | |
2484 | createPrivateNonConstGlobalForString(*F.getParent(), | |
2485 | StackDescription.str()); | |
1a4d82fc JJ |
2486 | |
2487 | IRB.CreateCall4(MS.MsanSetAllocaOrigin4Fn, | |
970d7e83 LB |
2488 | IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), |
2489 | ConstantInt::get(MS.IntptrTy, Size), | |
1a4d82fc JJ |
2490 | IRB.CreatePointerCast(Descr, IRB.getInt8PtrTy()), |
2491 | IRB.CreatePointerCast(&F, MS.IntptrTy)); | |
970d7e83 LB |
2492 | } |
2493 | } | |
2494 | ||
2495 | void visitSelectInst(SelectInst& I) { | |
2496 | IRBuilder<> IRB(&I); | |
1a4d82fc JJ |
2497 | // a = select b, c, d |
2498 | Value *B = I.getCondition(); | |
2499 | Value *C = I.getTrueValue(); | |
2500 | Value *D = I.getFalseValue(); | |
2501 | Value *Sb = getShadow(B); | |
2502 | Value *Sc = getShadow(C); | |
2503 | Value *Sd = getShadow(D); | |
2504 | ||
2505 | // Result shadow if condition shadow is 0. | |
2506 | Value *Sa0 = IRB.CreateSelect(B, Sc, Sd); | |
2507 | Value *Sa1; | |
2508 | if (I.getType()->isAggregateType()) { | |
2509 | // To avoid "sign extending" i1 to an arbitrary aggregate type, we just do | |
2510 | // an extra "select". This results in much more compact IR. | |
2511 | // Sa = select Sb, poisoned, (select b, Sc, Sd) | |
2512 | Sa1 = getPoisonedShadow(getShadowTy(I.getType())); | |
2513 | } else { | |
2514 | // Sa = select Sb, [ (c^d) | Sc | Sd ], [ b ? Sc : Sd ] | |
2515 | // If Sb (condition is poisoned), look for bits in c and d that are equal | |
2516 | // and both unpoisoned. | |
2517 | // If !Sb (condition is unpoisoned), simply pick one of Sc and Sd. | |
2518 | ||
2519 | // Cast arguments to shadow-compatible type. | |
2520 | C = CreateAppToShadowCast(IRB, C); | |
2521 | D = CreateAppToShadowCast(IRB, D); | |
2522 | ||
2523 | // Result shadow if condition shadow is 1. | |
2524 | Sa1 = IRB.CreateOr(IRB.CreateXor(C, D), IRB.CreateOr(Sc, Sd)); | |
2525 | } | |
2526 | Value *Sa = IRB.CreateSelect(Sb, Sa1, Sa0, "_msprop_select"); | |
2527 | setShadow(&I, Sa); | |
970d7e83 LB |
2528 | if (MS.TrackOrigins) { |
2529 | // Origins are always i32, so any vector conditions must be flattened. | |
2530 | // FIXME: consider tracking vector origins for app vectors? | |
1a4d82fc JJ |
2531 | if (B->getType()->isVectorTy()) { |
2532 | Type *FlatTy = getShadowTyNoVec(B->getType()); | |
2533 | B = IRB.CreateICmpNE(IRB.CreateBitCast(B, FlatTy), | |
2534 | ConstantInt::getNullValue(FlatTy)); | |
2535 | Sb = IRB.CreateICmpNE(IRB.CreateBitCast(Sb, FlatTy), | |
2536 | ConstantInt::getNullValue(FlatTy)); | |
970d7e83 | 2537 | } |
1a4d82fc JJ |
2538 | // a = select b, c, d |
2539 | // Oa = Sb ? Ob : (b ? Oc : Od) | |
85aaf69f SL |
2540 | setOrigin( |
2541 | &I, IRB.CreateSelect(Sb, getOrigin(I.getCondition()), | |
2542 | IRB.CreateSelect(B, getOrigin(I.getTrueValue()), | |
2543 | getOrigin(I.getFalseValue())))); | |
970d7e83 LB |
2544 | } |
2545 | } | |
2546 | ||
2547 | void visitLandingPadInst(LandingPadInst &I) { | |
2548 | // Do nothing. | |
2549 | // See http://code.google.com/p/memory-sanitizer/issues/detail?id=1 | |
2550 | setShadow(&I, getCleanShadow(&I)); | |
2551 | setOrigin(&I, getCleanOrigin()); | |
2552 | } | |
2553 | ||
2554 | void visitGetElementPtrInst(GetElementPtrInst &I) { | |
2555 | handleShadowOr(I); | |
2556 | } | |
2557 | ||
2558 | void visitExtractValueInst(ExtractValueInst &I) { | |
2559 | IRBuilder<> IRB(&I); | |
2560 | Value *Agg = I.getAggregateOperand(); | |
2561 | DEBUG(dbgs() << "ExtractValue: " << I << "\n"); | |
2562 | Value *AggShadow = getShadow(Agg); | |
2563 | DEBUG(dbgs() << " AggShadow: " << *AggShadow << "\n"); | |
2564 | Value *ResShadow = IRB.CreateExtractValue(AggShadow, I.getIndices()); | |
2565 | DEBUG(dbgs() << " ResShadow: " << *ResShadow << "\n"); | |
2566 | setShadow(&I, ResShadow); | |
1a4d82fc | 2567 | setOriginForNaryOp(I); |
970d7e83 LB |
2568 | } |
2569 | ||
2570 | void visitInsertValueInst(InsertValueInst &I) { | |
2571 | IRBuilder<> IRB(&I); | |
2572 | DEBUG(dbgs() << "InsertValue: " << I << "\n"); | |
2573 | Value *AggShadow = getShadow(I.getAggregateOperand()); | |
2574 | Value *InsShadow = getShadow(I.getInsertedValueOperand()); | |
2575 | DEBUG(dbgs() << " AggShadow: " << *AggShadow << "\n"); | |
2576 | DEBUG(dbgs() << " InsShadow: " << *InsShadow << "\n"); | |
2577 | Value *Res = IRB.CreateInsertValue(AggShadow, InsShadow, I.getIndices()); | |
2578 | DEBUG(dbgs() << " Res: " << *Res << "\n"); | |
2579 | setShadow(&I, Res); | |
1a4d82fc | 2580 | setOriginForNaryOp(I); |
970d7e83 LB |
2581 | } |
2582 | ||
2583 | void dumpInst(Instruction &I) { | |
2584 | if (CallInst *CI = dyn_cast<CallInst>(&I)) { | |
2585 | errs() << "ZZZ call " << CI->getCalledFunction()->getName() << "\n"; | |
2586 | } else { | |
2587 | errs() << "ZZZ " << I.getOpcodeName() << "\n"; | |
2588 | } | |
2589 | errs() << "QQQ " << I << "\n"; | |
2590 | } | |
2591 | ||
2592 | void visitResumeInst(ResumeInst &I) { | |
2593 | DEBUG(dbgs() << "Resume: " << I << "\n"); | |
2594 | // Nothing to do here. | |
2595 | } | |
2596 | ||
2597 | void visitInstruction(Instruction &I) { | |
2598 | // Everything else: stop propagating and check for poisoned shadow. | |
2599 | if (ClDumpStrictInstructions) | |
2600 | dumpInst(I); | |
2601 | DEBUG(dbgs() << "DEFAULT: " << I << "\n"); | |
2602 | for (size_t i = 0, n = I.getNumOperands(); i < n; i++) | |
1a4d82fc | 2603 | insertShadowCheck(I.getOperand(i), &I); |
970d7e83 LB |
2604 | setShadow(&I, getCleanShadow(&I)); |
2605 | setOrigin(&I, getCleanOrigin()); | |
2606 | } | |
2607 | }; | |
2608 | ||
2609 | /// \brief AMD64-specific implementation of VarArgHelper. | |
2610 | struct VarArgAMD64Helper : public VarArgHelper { | |
2611 | // An unfortunate workaround for asymmetric lowering of va_arg stuff. | |
2612 | // See a comment in visitCallSite for more details. | |
2613 | static const unsigned AMD64GpEndOffset = 48; // AMD64 ABI Draft 0.99.6 p3.5.7 | |
2614 | static const unsigned AMD64FpEndOffset = 176; | |
2615 | ||
2616 | Function &F; | |
2617 | MemorySanitizer &MS; | |
2618 | MemorySanitizerVisitor &MSV; | |
2619 | Value *VAArgTLSCopy; | |
2620 | Value *VAArgOverflowSize; | |
2621 | ||
2622 | SmallVector<CallInst*, 16> VAStartInstrumentationList; | |
2623 | ||
2624 | VarArgAMD64Helper(Function &F, MemorySanitizer &MS, | |
2625 | MemorySanitizerVisitor &MSV) | |
1a4d82fc JJ |
2626 | : F(F), MS(MS), MSV(MSV), VAArgTLSCopy(nullptr), |
2627 | VAArgOverflowSize(nullptr) {} | |
970d7e83 LB |
2628 | |
2629 | enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory }; | |
2630 | ||
2631 | ArgKind classifyArgument(Value* arg) { | |
2632 | // A very rough approximation of X86_64 argument classification rules. | |
2633 | Type *T = arg->getType(); | |
2634 | if (T->isFPOrFPVectorTy() || T->isX86_MMXTy()) | |
2635 | return AK_FloatingPoint; | |
2636 | if (T->isIntegerTy() && T->getPrimitiveSizeInBits() <= 64) | |
2637 | return AK_GeneralPurpose; | |
2638 | if (T->isPointerTy()) | |
2639 | return AK_GeneralPurpose; | |
2640 | return AK_Memory; | |
2641 | } | |
2642 | ||
2643 | // For VarArg functions, store the argument shadow in an ABI-specific format | |
2644 | // that corresponds to va_list layout. | |
2645 | // We do this because Clang lowers va_arg in the frontend, and this pass | |
2646 | // only sees the low level code that deals with va_list internals. | |
2647 | // A much easier alternative (provided that Clang emits va_arg instructions) | |
2648 | // would have been to associate each live instance of va_list with a copy of | |
2649 | // MSanParamTLS, and extract shadow on va_arg() call in the argument list | |
2650 | // order. | |
1a4d82fc | 2651 | void visitCallSite(CallSite &CS, IRBuilder<> &IRB) override { |
970d7e83 LB |
2652 | unsigned GpOffset = 0; |
2653 | unsigned FpOffset = AMD64GpEndOffset; | |
2654 | unsigned OverflowOffset = AMD64FpEndOffset; | |
2655 | for (CallSite::arg_iterator ArgIt = CS.arg_begin(), End = CS.arg_end(); | |
2656 | ArgIt != End; ++ArgIt) { | |
2657 | Value *A = *ArgIt; | |
1a4d82fc JJ |
2658 | unsigned ArgNo = CS.getArgumentNo(ArgIt); |
2659 | bool IsByVal = CS.paramHasAttr(ArgNo + 1, Attribute::ByVal); | |
2660 | if (IsByVal) { | |
2661 | // ByVal arguments always go to the overflow area. | |
2662 | assert(A->getType()->isPointerTy()); | |
2663 | Type *RealTy = A->getType()->getPointerElementType(); | |
2664 | uint64_t ArgSize = MS.DL->getTypeAllocSize(RealTy); | |
2665 | Value *Base = getShadowPtrForVAArgument(RealTy, IRB, OverflowOffset); | |
85aaf69f | 2666 | OverflowOffset += RoundUpToAlignment(ArgSize, 8); |
1a4d82fc JJ |
2667 | IRB.CreateMemCpy(Base, MSV.getShadowPtr(A, IRB.getInt8Ty(), IRB), |
2668 | ArgSize, kShadowTLSAlignment); | |
2669 | } else { | |
2670 | ArgKind AK = classifyArgument(A); | |
2671 | if (AK == AK_GeneralPurpose && GpOffset >= AMD64GpEndOffset) | |
2672 | AK = AK_Memory; | |
2673 | if (AK == AK_FloatingPoint && FpOffset >= AMD64FpEndOffset) | |
2674 | AK = AK_Memory; | |
2675 | Value *Base; | |
2676 | switch (AK) { | |
2677 | case AK_GeneralPurpose: | |
2678 | Base = getShadowPtrForVAArgument(A->getType(), IRB, GpOffset); | |
2679 | GpOffset += 8; | |
2680 | break; | |
2681 | case AK_FloatingPoint: | |
2682 | Base = getShadowPtrForVAArgument(A->getType(), IRB, FpOffset); | |
2683 | FpOffset += 16; | |
2684 | break; | |
2685 | case AK_Memory: | |
2686 | uint64_t ArgSize = MS.DL->getTypeAllocSize(A->getType()); | |
2687 | Base = getShadowPtrForVAArgument(A->getType(), IRB, OverflowOffset); | |
85aaf69f | 2688 | OverflowOffset += RoundUpToAlignment(ArgSize, 8); |
1a4d82fc JJ |
2689 | } |
2690 | IRB.CreateAlignedStore(MSV.getShadow(A), Base, kShadowTLSAlignment); | |
970d7e83 | 2691 | } |
970d7e83 LB |
2692 | } |
2693 | Constant *OverflowSize = | |
2694 | ConstantInt::get(IRB.getInt64Ty(), OverflowOffset - AMD64FpEndOffset); | |
2695 | IRB.CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS); | |
2696 | } | |
2697 | ||
2698 | /// \brief Compute the shadow address for a given va_arg. | |
1a4d82fc | 2699 | Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB, |
970d7e83 LB |
2700 | int ArgOffset) { |
2701 | Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy); | |
2702 | Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset)); | |
1a4d82fc | 2703 | return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0), |
970d7e83 LB |
2704 | "_msarg"); |
2705 | } | |
2706 | ||
1a4d82fc | 2707 | void visitVAStartInst(VAStartInst &I) override { |
970d7e83 LB |
2708 | IRBuilder<> IRB(&I); |
2709 | VAStartInstrumentationList.push_back(&I); | |
2710 | Value *VAListTag = I.getArgOperand(0); | |
2711 | Value *ShadowPtr = MSV.getShadowPtr(VAListTag, IRB.getInt8Ty(), IRB); | |
2712 | ||
2713 | // Unpoison the whole __va_list_tag. | |
2714 | // FIXME: magic ABI constants. | |
2715 | IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()), | |
2716 | /* size */24, /* alignment */8, false); | |
2717 | } | |
2718 | ||
1a4d82fc | 2719 | void visitVACopyInst(VACopyInst &I) override { |
970d7e83 LB |
2720 | IRBuilder<> IRB(&I); |
2721 | Value *VAListTag = I.getArgOperand(0); | |
2722 | Value *ShadowPtr = MSV.getShadowPtr(VAListTag, IRB.getInt8Ty(), IRB); | |
2723 | ||
2724 | // Unpoison the whole __va_list_tag. | |
2725 | // FIXME: magic ABI constants. | |
2726 | IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()), | |
2727 | /* size */24, /* alignment */8, false); | |
2728 | } | |
2729 | ||
1a4d82fc | 2730 | void finalizeInstrumentation() override { |
970d7e83 LB |
2731 | assert(!VAArgOverflowSize && !VAArgTLSCopy && |
2732 | "finalizeInstrumentation called twice"); | |
2733 | if (!VAStartInstrumentationList.empty()) { | |
2734 | // If there is a va_start in this function, make a backup copy of | |
2735 | // va_arg_tls somewhere in the function entry block. | |
2736 | IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI()); | |
2737 | VAArgOverflowSize = IRB.CreateLoad(MS.VAArgOverflowSizeTLS); | |
2738 | Value *CopySize = | |
2739 | IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, AMD64FpEndOffset), | |
2740 | VAArgOverflowSize); | |
2741 | VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize); | |
2742 | IRB.CreateMemCpy(VAArgTLSCopy, MS.VAArgTLS, CopySize, 8); | |
2743 | } | |
2744 | ||
2745 | // Instrument va_start. | |
2746 | // Copy va_list shadow from the backup copy of the TLS contents. | |
2747 | for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) { | |
2748 | CallInst *OrigInst = VAStartInstrumentationList[i]; | |
2749 | IRBuilder<> IRB(OrigInst->getNextNode()); | |
2750 | Value *VAListTag = OrigInst->getArgOperand(0); | |
2751 | ||
2752 | Value *RegSaveAreaPtrPtr = | |
2753 | IRB.CreateIntToPtr( | |
2754 | IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy), | |
2755 | ConstantInt::get(MS.IntptrTy, 16)), | |
2756 | Type::getInt64PtrTy(*MS.C)); | |
2757 | Value *RegSaveAreaPtr = IRB.CreateLoad(RegSaveAreaPtrPtr); | |
2758 | Value *RegSaveAreaShadowPtr = | |
2759 | MSV.getShadowPtr(RegSaveAreaPtr, IRB.getInt8Ty(), IRB); | |
2760 | IRB.CreateMemCpy(RegSaveAreaShadowPtr, VAArgTLSCopy, | |
2761 | AMD64FpEndOffset, 16); | |
2762 | ||
2763 | Value *OverflowArgAreaPtrPtr = | |
2764 | IRB.CreateIntToPtr( | |
2765 | IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy), | |
2766 | ConstantInt::get(MS.IntptrTy, 8)), | |
2767 | Type::getInt64PtrTy(*MS.C)); | |
2768 | Value *OverflowArgAreaPtr = IRB.CreateLoad(OverflowArgAreaPtrPtr); | |
2769 | Value *OverflowArgAreaShadowPtr = | |
2770 | MSV.getShadowPtr(OverflowArgAreaPtr, IRB.getInt8Ty(), IRB); | |
1a4d82fc | 2771 | Value *SrcPtr = IRB.CreateConstGEP1_32(VAArgTLSCopy, AMD64FpEndOffset); |
970d7e83 LB |
2772 | IRB.CreateMemCpy(OverflowArgAreaShadowPtr, SrcPtr, VAArgOverflowSize, 16); |
2773 | } | |
2774 | } | |
2775 | }; | |
2776 | ||
1a4d82fc JJ |
2777 | /// \brief A no-op implementation of VarArgHelper. |
2778 | struct VarArgNoOpHelper : public VarArgHelper { | |
2779 | VarArgNoOpHelper(Function &F, MemorySanitizer &MS, | |
2780 | MemorySanitizerVisitor &MSV) {} | |
2781 | ||
2782 | void visitCallSite(CallSite &CS, IRBuilder<> &IRB) override {} | |
2783 | ||
2784 | void visitVAStartInst(VAStartInst &I) override {} | |
2785 | ||
2786 | void visitVACopyInst(VACopyInst &I) override {} | |
2787 | ||
2788 | void finalizeInstrumentation() override {} | |
2789 | }; | |
2790 | ||
2791 | VarArgHelper *CreateVarArgHelper(Function &Func, MemorySanitizer &Msan, | |
970d7e83 | 2792 | MemorySanitizerVisitor &Visitor) { |
1a4d82fc JJ |
2793 | // VarArg handling is only implemented on AMD64. False positives are possible |
2794 | // on other platforms. | |
2795 | llvm::Triple TargetTriple(Func.getParent()->getTargetTriple()); | |
2796 | if (TargetTriple.getArch() == llvm::Triple::x86_64) | |
2797 | return new VarArgAMD64Helper(Func, Msan, Visitor); | |
2798 | else | |
2799 | return new VarArgNoOpHelper(Func, Msan, Visitor); | |
970d7e83 LB |
2800 | } |
2801 | ||
2802 | } // namespace | |
2803 | ||
2804 | bool MemorySanitizer::runOnFunction(Function &F) { | |
2805 | MemorySanitizerVisitor Visitor(F, *this); | |
2806 | ||
2807 | // Clear out readonly/readnone attributes. | |
2808 | AttrBuilder B; | |
2809 | B.addAttribute(Attribute::ReadOnly) | |
2810 | .addAttribute(Attribute::ReadNone); | |
2811 | F.removeAttributes(AttributeSet::FunctionIndex, | |
2812 | AttributeSet::get(F.getContext(), | |
2813 | AttributeSet::FunctionIndex, B)); | |
2814 | ||
2815 | return Visitor.runOnFunction(); | |
2816 | } |