]>
Commit | Line | Data |
---|---|---|
7cac9316 XL |
1 | //===-- xray_interface.cpp --------------------------------------*- C++ -*-===// |
2 | // | |
3 | // The LLVM Compiler Infrastructure | |
4 | // | |
5 | // This file is distributed under the University of Illinois Open Source | |
6 | // License. See LICENSE.TXT for details. | |
7 | // | |
8 | //===----------------------------------------------------------------------===// | |
9 | // | |
10 | // This file is a part of XRay, a dynamic runtime instrumentation system. | |
11 | // | |
12 | // Implementation of the API functions. | |
13 | // | |
14 | //===----------------------------------------------------------------------===// | |
15 | ||
16 | #include "xray_interface_internal.h" | |
17 | ||
7cac9316 XL |
18 | #include <cstdint> |
19 | #include <cstdio> | |
20 | #include <errno.h> | |
21 | #include <limits> | |
22 | #include <sys/mman.h> | |
23 | ||
24 | #include "sanitizer_common/sanitizer_common.h" | |
25 | #include "xray_defs.h" | |
2c00a5a8 XL |
26 | #include "xray_flags.h" |
27 | ||
28 | extern __sanitizer::SpinMutex XRayInstrMapMutex; | |
29 | extern __sanitizer::atomic_uint8_t XRayInitialized; | |
30 | extern __xray::XRaySledMap XRayInstrMap; | |
7cac9316 XL |
31 | |
32 | namespace __xray { | |
33 | ||
34 | #if defined(__x86_64__) | |
7cac9316 XL |
35 | static const int16_t cSledLength = 12; |
36 | #elif defined(__aarch64__) | |
37 | static const int16_t cSledLength = 32; | |
38 | #elif defined(__arm__) | |
39 | static const int16_t cSledLength = 28; | |
2c00a5a8 XL |
40 | #elif SANITIZER_MIPS32 |
41 | static const int16_t cSledLength = 48; | |
42 | #elif SANITIZER_MIPS64 | |
43 | static const int16_t cSledLength = 64; | |
44 | #elif defined(__powerpc64__) | |
45 | static const int16_t cSledLength = 8; | |
7cac9316 XL |
46 | #else |
47 | #error "Unsupported CPU Architecture" | |
48 | #endif /* CPU architecture */ | |
49 | ||
50 | // This is the function to call when we encounter the entry or exit sleds. | |
2c00a5a8 XL |
51 | __sanitizer::atomic_uintptr_t XRayPatchedFunction{0}; |
52 | ||
53 | // This is the function to call from the arg1-enabled sleds/trampolines. | |
54 | __sanitizer::atomic_uintptr_t XRayArgLogger{0}; | |
55 | ||
56 | // This is the function to call when we encounter a custom event log call. | |
57 | __sanitizer::atomic_uintptr_t XRayPatchedCustomEvent{0}; | |
58 | ||
59 | // This is the global status to determine whether we are currently | |
60 | // patching/unpatching. | |
61 | __sanitizer::atomic_uint8_t XRayPatching{0}; | |
7cac9316 XL |
62 | |
63 | // MProtectHelper is an RAII wrapper for calls to mprotect(...) that will undo | |
64 | // any successful mprotect(...) changes. This is used to make a page writeable | |
65 | // and executable, and upon destruction if it was successful in doing so returns | |
66 | // the page into a read-only and executable page. | |
67 | // | |
68 | // This is only used specifically for runtime-patching of the XRay | |
69 | // instrumentation points. This assumes that the executable pages are originally | |
70 | // read-and-execute only. | |
71 | class MProtectHelper { | |
72 | void *PageAlignedAddr; | |
73 | std::size_t MProtectLen; | |
74 | bool MustCleanup; | |
75 | ||
76 | public: | |
77 | explicit MProtectHelper(void *PageAlignedAddr, | |
78 | std::size_t MProtectLen) XRAY_NEVER_INSTRUMENT | |
79 | : PageAlignedAddr(PageAlignedAddr), | |
80 | MProtectLen(MProtectLen), | |
81 | MustCleanup(false) {} | |
82 | ||
83 | int MakeWriteable() XRAY_NEVER_INSTRUMENT { | |
84 | auto R = mprotect(PageAlignedAddr, MProtectLen, | |
85 | PROT_READ | PROT_WRITE | PROT_EXEC); | |
86 | if (R != -1) | |
87 | MustCleanup = true; | |
88 | return R; | |
89 | } | |
90 | ||
91 | ~MProtectHelper() XRAY_NEVER_INSTRUMENT { | |
92 | if (MustCleanup) { | |
93 | mprotect(PageAlignedAddr, MProtectLen, PROT_READ | PROT_EXEC); | |
94 | } | |
95 | } | |
96 | }; | |
97 | ||
2c00a5a8 | 98 | namespace { |
7cac9316 | 99 | |
2c00a5a8 XL |
100 | bool patchSled(const XRaySledEntry &Sled, bool Enable, |
101 | int32_t FuncId) XRAY_NEVER_INSTRUMENT { | |
102 | bool Success = false; | |
103 | switch (Sled.Kind) { | |
104 | case XRayEntryType::ENTRY: | |
105 | Success = patchFunctionEntry(Enable, FuncId, Sled, __xray_FunctionEntry); | |
106 | break; | |
107 | case XRayEntryType::EXIT: | |
108 | Success = patchFunctionExit(Enable, FuncId, Sled); | |
109 | break; | |
110 | case XRayEntryType::TAIL: | |
111 | Success = patchFunctionTailExit(Enable, FuncId, Sled); | |
112 | break; | |
113 | case XRayEntryType::LOG_ARGS_ENTRY: | |
114 | Success = patchFunctionEntry(Enable, FuncId, Sled, __xray_ArgLoggerEntry); | |
115 | break; | |
116 | case XRayEntryType::CUSTOM_EVENT: | |
117 | Success = patchCustomEvent(Enable, FuncId, Sled); | |
118 | break; | |
119 | default: | |
120 | Report("Unsupported sled kind '%d' @%04x\n", Sled.Address, int(Sled.Kind)); | |
121 | return false; | |
7cac9316 | 122 | } |
2c00a5a8 | 123 | return Success; |
7cac9316 XL |
124 | } |
125 | ||
2c00a5a8 XL |
126 | XRayPatchingStatus patchFunction(int32_t FuncId, |
127 | bool Enable) XRAY_NEVER_INSTRUMENT { | |
128 | if (!__sanitizer::atomic_load(&XRayInitialized, | |
129 | __sanitizer::memory_order_acquire)) | |
130 | return XRayPatchingStatus::NOT_INITIALIZED; // Not initialized. | |
7cac9316 | 131 | |
2c00a5a8 XL |
132 | uint8_t NotPatching = false; |
133 | if (!__sanitizer::atomic_compare_exchange_strong( | |
134 | &XRayPatching, &NotPatching, true, __sanitizer::memory_order_acq_rel)) | |
135 | return XRayPatchingStatus::ONGOING; // Already patching. | |
7cac9316 | 136 | |
2c00a5a8 XL |
137 | // Next, we look for the function index. |
138 | XRaySledMap InstrMap; | |
139 | { | |
140 | __sanitizer::SpinMutexLock Guard(&XRayInstrMapMutex); | |
141 | InstrMap = XRayInstrMap; | |
142 | } | |
7cac9316 | 143 | |
2c00a5a8 XL |
144 | // If we don't have an index, we can't patch individual functions. |
145 | if (InstrMap.Functions == 0) | |
146 | return XRayPatchingStatus::NOT_INITIALIZED; | |
7cac9316 | 147 | |
2c00a5a8 XL |
148 | // FuncId must be a positive number, less than the number of functions |
149 | // instrumented. | |
150 | if (FuncId <= 0 || static_cast<size_t>(FuncId) > InstrMap.Functions) { | |
151 | Report("Invalid function id provided: %d\n", FuncId); | |
152 | return XRayPatchingStatus::FAILED; | |
153 | } | |
154 | ||
155 | // Now we patch ths sleds for this specific function. | |
156 | auto SledRange = InstrMap.SledsIndex[FuncId - 1]; | |
157 | auto *f = SledRange.Begin; | |
158 | auto *e = SledRange.End; | |
159 | ||
160 | bool SucceedOnce = false; | |
161 | while (f != e) | |
162 | SucceedOnce |= patchSled(*f++, Enable, FuncId); | |
163 | ||
164 | __sanitizer::atomic_store(&XRayPatching, false, | |
165 | __sanitizer::memory_order_release); | |
7cac9316 | 166 | |
2c00a5a8 XL |
167 | if (!SucceedOnce) { |
168 | Report("Failed patching any sled for function '%d'.", FuncId); | |
169 | return XRayPatchingStatus::FAILED; | |
170 | } | |
171 | ||
172 | return XRayPatchingStatus::SUCCESS; | |
7cac9316 XL |
173 | } |
174 | ||
2c00a5a8 | 175 | // controlPatching implements the common internals of the patching/unpatching |
7cac9316 XL |
176 | // implementation. |Enable| defines whether we're enabling or disabling the |
177 | // runtime XRay instrumentation. | |
2c00a5a8 XL |
178 | XRayPatchingStatus controlPatching(bool Enable) XRAY_NEVER_INSTRUMENT { |
179 | if (!__sanitizer::atomic_load(&XRayInitialized, | |
180 | __sanitizer::memory_order_acquire)) | |
7cac9316 XL |
181 | return XRayPatchingStatus::NOT_INITIALIZED; // Not initialized. |
182 | ||
2c00a5a8 XL |
183 | uint8_t NotPatching = false; |
184 | if (!__sanitizer::atomic_compare_exchange_strong( | |
185 | &XRayPatching, &NotPatching, true, __sanitizer::memory_order_acq_rel)) | |
7cac9316 | 186 | return XRayPatchingStatus::ONGOING; // Already patching. |
7cac9316 | 187 | |
2c00a5a8 XL |
188 | uint8_t PatchingSuccess = false; |
189 | auto XRayPatchingStatusResetter = | |
190 | __sanitizer::at_scope_exit([&PatchingSuccess] { | |
191 | if (!PatchingSuccess) | |
192 | __sanitizer::atomic_store(&XRayPatching, false, | |
193 | __sanitizer::memory_order_release); | |
194 | }); | |
7cac9316 | 195 | |
2c00a5a8 XL |
196 | XRaySledMap InstrMap; |
197 | { | |
198 | __sanitizer::SpinMutexLock Guard(&XRayInstrMapMutex); | |
199 | InstrMap = XRayInstrMap; | |
200 | } | |
7cac9316 XL |
201 | if (InstrMap.Entries == 0) |
202 | return XRayPatchingStatus::NOT_INITIALIZED; | |
203 | ||
2c00a5a8 XL |
204 | uint32_t FuncId = 1; |
205 | uint64_t CurFun = 0; | |
206 | ||
207 | // First we want to find the bounds for which we have instrumentation points, | |
208 | // and try to get as few calls to mprotect(...) as possible. We're assuming | |
209 | // that all the sleds for the instrumentation map are contiguous as a single | |
210 | // set of pages. When we do support dynamic shared object instrumentation, | |
211 | // we'll need to do this for each set of page load offsets per DSO loaded. For | |
212 | // now we're assuming we can mprotect the whole section of text between the | |
213 | // minimum sled address and the maximum sled address (+ the largest sled | |
214 | // size). | |
215 | auto MinSled = InstrMap.Sleds[0]; | |
216 | auto MaxSled = InstrMap.Sleds[InstrMap.Entries - 1]; | |
217 | for (std::size_t I = 0; I < InstrMap.Entries; I++) { | |
218 | const auto &Sled = InstrMap.Sleds[I]; | |
219 | if (Sled.Address < MinSled.Address) | |
220 | MinSled = Sled; | |
221 | if (Sled.Address > MaxSled.Address) | |
222 | MaxSled = Sled; | |
223 | } | |
224 | ||
225 | const size_t PageSize = flags()->xray_page_size_override > 0 | |
226 | ? flags()->xray_page_size_override | |
227 | : GetPageSizeCached(); | |
7cac9316 XL |
228 | if ((PageSize == 0) || ((PageSize & (PageSize - 1)) != 0)) { |
229 | Report("System page size is not a power of two: %lld\n", PageSize); | |
230 | return XRayPatchingStatus::FAILED; | |
231 | } | |
232 | ||
2c00a5a8 XL |
233 | void *PageAlignedAddr = |
234 | reinterpret_cast<void *>(MinSled.Address & ~(PageSize - 1)); | |
235 | size_t MProtectLen = | |
236 | (MaxSled.Address - reinterpret_cast<uptr>(PageAlignedAddr)) + cSledLength; | |
237 | MProtectHelper Protector(PageAlignedAddr, MProtectLen); | |
238 | if (Protector.MakeWriteable() == -1) { | |
239 | Report("Failed mprotect: %d\n", errno); | |
240 | return XRayPatchingStatus::FAILED; | |
241 | } | |
242 | ||
243 | for (std::size_t I = 0; I < InstrMap.Entries; ++I) { | |
244 | auto &Sled = InstrMap.Sleds[I]; | |
7cac9316 XL |
245 | auto F = Sled.Function; |
246 | if (CurFun == 0) | |
247 | CurFun = F; | |
248 | if (F != CurFun) { | |
249 | ++FuncId; | |
250 | CurFun = F; | |
251 | } | |
2c00a5a8 | 252 | patchSled(Sled, Enable, FuncId); |
7cac9316 | 253 | } |
2c00a5a8 XL |
254 | __sanitizer::atomic_store(&XRayPatching, false, |
255 | __sanitizer::memory_order_release); | |
7cac9316 XL |
256 | PatchingSuccess = true; |
257 | return XRayPatchingStatus::SUCCESS; | |
258 | } | |
259 | ||
2c00a5a8 XL |
260 | XRayPatchingStatus mprotectAndPatchFunction(int32_t FuncId, |
261 | bool Enable) XRAY_NEVER_INSTRUMENT { | |
262 | XRaySledMap InstrMap; | |
263 | { | |
264 | __sanitizer::SpinMutexLock Guard(&XRayInstrMapMutex); | |
265 | InstrMap = XRayInstrMap; | |
266 | } | |
267 | ||
268 | // FuncId must be a positive number, less than the number of functions | |
269 | // instrumented. | |
270 | if (FuncId <= 0 || static_cast<size_t>(FuncId) > InstrMap.Functions) { | |
271 | Report("Invalid function id provided: %d\n", FuncId); | |
272 | return XRayPatchingStatus::FAILED; | |
273 | } | |
274 | ||
275 | const size_t PageSize = flags()->xray_page_size_override > 0 | |
276 | ? flags()->xray_page_size_override | |
277 | : GetPageSizeCached(); | |
278 | if ((PageSize == 0) || ((PageSize & (PageSize - 1)) != 0)) { | |
279 | Report("Provided page size is not a power of two: %lld\n", PageSize); | |
280 | return XRayPatchingStatus::FAILED; | |
281 | } | |
282 | ||
283 | // Here we compute the minumum sled and maximum sled associated with a | |
284 | // particular function ID. | |
285 | auto SledRange = InstrMap.SledsIndex[FuncId - 1]; | |
286 | auto *f = SledRange.Begin; | |
287 | auto *e = SledRange.End; | |
288 | auto MinSled = *f; | |
289 | auto MaxSled = *(SledRange.End - 1); | |
290 | while (f != e) { | |
291 | if (f->Address < MinSled.Address) | |
292 | MinSled = *f; | |
293 | if (f->Address > MaxSled.Address) | |
294 | MaxSled = *f; | |
295 | ++f; | |
296 | } | |
297 | ||
298 | void *PageAlignedAddr = | |
299 | reinterpret_cast<void *>(MinSled.Address & ~(PageSize - 1)); | |
300 | size_t MProtectLen = | |
301 | (MaxSled.Address - reinterpret_cast<uptr>(PageAlignedAddr)) + cSledLength; | |
302 | MProtectHelper Protector(PageAlignedAddr, MProtectLen); | |
303 | if (Protector.MakeWriteable() == -1) { | |
304 | Report("Failed mprotect: %d\n", errno); | |
305 | return XRayPatchingStatus::FAILED; | |
306 | } | |
307 | return patchFunction(FuncId, Enable); | |
308 | } | |
309 | ||
310 | } // namespace | |
311 | ||
312 | } // namespace __xray | |
313 | ||
314 | using namespace __xray; | |
315 | ||
316 | // The following functions are declared `extern "C" {...}` in the header, hence | |
317 | // they're defined in the global namespace. | |
318 | ||
319 | int __xray_set_handler(void (*entry)(int32_t, | |
320 | XRayEntryType)) XRAY_NEVER_INSTRUMENT { | |
321 | if (__sanitizer::atomic_load(&XRayInitialized, | |
322 | __sanitizer::memory_order_acquire)) { | |
323 | ||
324 | __sanitizer::atomic_store(&__xray::XRayPatchedFunction, | |
325 | reinterpret_cast<uintptr_t>(entry), | |
326 | __sanitizer::memory_order_release); | |
327 | return 1; | |
328 | } | |
329 | return 0; | |
330 | } | |
331 | ||
332 | int __xray_set_customevent_handler(void (*entry)(void *, size_t)) | |
333 | XRAY_NEVER_INSTRUMENT { | |
334 | if (__sanitizer::atomic_load(&XRayInitialized, | |
335 | __sanitizer::memory_order_acquire)) { | |
336 | __sanitizer::atomic_store(&__xray::XRayPatchedCustomEvent, | |
337 | reinterpret_cast<uintptr_t>(entry), | |
338 | __sanitizer::memory_order_release); | |
339 | return 1; | |
340 | } | |
341 | return 0; | |
342 | } | |
343 | ||
344 | int __xray_remove_handler() XRAY_NEVER_INSTRUMENT { | |
345 | return __xray_set_handler(nullptr); | |
346 | } | |
347 | ||
348 | int __xray_remove_customevent_handler() XRAY_NEVER_INSTRUMENT { | |
349 | return __xray_set_customevent_handler(nullptr); | |
350 | } | |
351 | ||
7cac9316 | 352 | XRayPatchingStatus __xray_patch() XRAY_NEVER_INSTRUMENT { |
2c00a5a8 | 353 | return controlPatching(true); |
7cac9316 XL |
354 | } |
355 | ||
356 | XRayPatchingStatus __xray_unpatch() XRAY_NEVER_INSTRUMENT { | |
2c00a5a8 XL |
357 | return controlPatching(false); |
358 | } | |
359 | ||
360 | XRayPatchingStatus __xray_patch_function(int32_t FuncId) XRAY_NEVER_INSTRUMENT { | |
361 | return mprotectAndPatchFunction(FuncId, true); | |
362 | } | |
363 | ||
364 | XRayPatchingStatus | |
365 | __xray_unpatch_function(int32_t FuncId) XRAY_NEVER_INSTRUMENT { | |
366 | return mprotectAndPatchFunction(FuncId, false); | |
367 | } | |
368 | ||
369 | int __xray_set_handler_arg1(void (*entry)(int32_t, XRayEntryType, uint64_t)) { | |
370 | if (!__sanitizer::atomic_load(&XRayInitialized, | |
371 | __sanitizer::memory_order_acquire)) | |
372 | return 0; | |
373 | ||
374 | // A relaxed write might not be visible even if the current thread gets | |
375 | // scheduled on a different CPU/NUMA node. We need to wait for everyone to | |
376 | // have this handler installed for consistency of collected data across CPUs. | |
377 | __sanitizer::atomic_store(&XRayArgLogger, reinterpret_cast<uint64_t>(entry), | |
378 | __sanitizer::memory_order_release); | |
379 | return 1; | |
380 | } | |
381 | ||
382 | int __xray_remove_handler_arg1() { return __xray_set_handler_arg1(nullptr); } | |
383 | ||
384 | uintptr_t __xray_function_address(int32_t FuncId) XRAY_NEVER_INSTRUMENT { | |
385 | __sanitizer::SpinMutexLock Guard(&XRayInstrMapMutex); | |
386 | if (FuncId <= 0 || static_cast<size_t>(FuncId) > XRayInstrMap.Functions) | |
387 | return 0; | |
388 | return XRayInstrMap.SledsIndex[FuncId - 1].Begin->Function | |
389 | // On PPC, function entries are always aligned to 16 bytes. The beginning of a | |
390 | // sled might be a local entry, which is always +8 based on the global entry. | |
391 | // Always return the global entry. | |
392 | #ifdef __PPC__ | |
393 | & ~0xf | |
394 | #endif | |
395 | ; | |
396 | } | |
397 | ||
398 | size_t __xray_max_function_id() XRAY_NEVER_INSTRUMENT { | |
399 | __sanitizer::SpinMutexLock Guard(&XRayInstrMapMutex); | |
400 | return XRayInstrMap.Functions; | |
7cac9316 | 401 | } |