]>
Commit | Line | Data |
---|---|---|
1a4d82fc JJ |
1 | #include <stdio.h> |
2 | ||
781aab86 | 3 | #include <cstddef> |
add651ee | 4 | #include <iomanip> |
041b39d2 | 5 | #include <vector> |
ff7c6d11 | 6 | #include <set> |
041b39d2 | 7 | |
1b1a35ee | 8 | #include "LLVMWrapper.h" |
970d7e83 | 9 | |
6a06907d | 10 | #include "llvm/Analysis/AliasAnalysis.h" |
62682a34 SL |
11 | #include "llvm/Analysis/TargetLibraryInfo.h" |
12 | #include "llvm/Analysis/TargetTransformInfo.h" | |
781aab86 | 13 | #include "llvm/CodeGen/CommandFlags.h" |
0731742a | 14 | #include "llvm/CodeGen/TargetSubtargetInfo.h" |
5bcae85e | 15 | #include "llvm/IR/AutoUpgrade.h" |
041b39d2 | 16 | #include "llvm/IR/AssemblyAnnotationWriter.h" |
0731742a | 17 | #include "llvm/IR/IntrinsicInst.h" |
74b04a01 | 18 | #include "llvm/IR/Verifier.h" |
9ffffee4 | 19 | #include "llvm/MC/TargetRegistry.h" |
f9f354fc XL |
20 | #include "llvm/Object/ObjectFile.h" |
21 | #include "llvm/Object/IRObjectFile.h" | |
74b04a01 | 22 | #include "llvm/Passes/PassBuilder.h" |
a2a8927a | 23 | #include "llvm/Passes/PassPlugin.h" |
74b04a01 | 24 | #include "llvm/Passes/StandardInstrumentations.h" |
32a655c1 SL |
25 | #include "llvm/Support/CBindingWrapping.h" |
26 | #include "llvm/Support/FileSystem.h" | |
9ffffee4 FG |
27 | #if LLVM_VERSION_GE(17, 0) |
28 | #include "llvm/Support/VirtualFileSystem.h" | |
3c0e092e | 29 | #endif |
c1a9b12d | 30 | #include "llvm/Target/TargetMachine.h" |
476ff2be | 31 | #include "llvm/Transforms/IPO/AlwaysInliner.h" |
ea8adc8c | 32 | #include "llvm/Transforms/IPO/FunctionImport.h" |
2b03887a | 33 | #include "llvm/Transforms/IPO/Internalize.h" |
add651ee | 34 | #include "llvm/Transforms/IPO/LowerTypeTests.h" |
f2b60f7d | 35 | #include "llvm/Transforms/IPO/ThinLTOBitcodeWriter.h" |
c295e0f8 | 36 | #include "llvm/Transforms/Utils/AddDiscriminators.h" |
ea8adc8c XL |
37 | #include "llvm/Transforms/Utils/FunctionImportUtils.h" |
38 | #include "llvm/LTO/LTO.h" | |
2b03887a | 39 | #include "llvm/Bitcode/BitcodeWriter.h" |
970d7e83 | 40 | |
60c5eb7d | 41 | #include "llvm/Transforms/Instrumentation.h" |
60c5eb7d | 42 | #include "llvm/Transforms/Instrumentation/AddressSanitizer.h" |
74b04a01 | 43 | #include "llvm/Support/TimeProfiler.h" |
17df50a5 XL |
44 | #include "llvm/Transforms/Instrumentation/GCOVProfiler.h" |
45 | #include "llvm/Transforms/Instrumentation/InstrProfiling.h" | |
60c5eb7d XL |
46 | #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h" |
47 | #include "llvm/Transforms/Instrumentation/MemorySanitizer.h" | |
6a06907d | 48 | #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h" |
74b04a01 | 49 | #include "llvm/Transforms/Utils/CanonicalizeAliases.h" |
74b04a01 | 50 | #include "llvm/Transforms/Utils/NameAnonGlobals.h" |
c295e0f8 | 51 | #include "llvm/Transforms/Utils.h" |
60c5eb7d | 52 | |
1a4d82fc | 53 | using namespace llvm; |
970d7e83 | 54 | |
781aab86 FG |
55 | static codegen::RegisterCodeGenFlags CGF; |
56 | ||
1a4d82fc JJ |
57 | typedef struct LLVMOpaquePass *LLVMPassRef; |
58 | typedef struct LLVMOpaqueTargetMachine *LLVMTargetMachineRef; | |
59 | ||
60 | DEFINE_STDCXX_CONVERSION_FUNCTIONS(Pass, LLVMPassRef) | |
61 | DEFINE_STDCXX_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef) | |
970d7e83 | 62 | |
74b04a01 | 63 | extern "C" void LLVMTimeTraceProfilerInitialize() { |
ba9703b0 XL |
64 | timeTraceProfilerInitialize( |
65 | /* TimeTraceGranularity */ 0, | |
66 | /* ProcName */ "rustc"); | |
74b04a01 XL |
67 | } |
68 | ||
3c0e092e XL |
69 | extern "C" void LLVMTimeTraceProfilerFinishThread() { |
70 | timeTraceProfilerFinishThread(); | |
71 | } | |
72 | ||
74b04a01 | 73 | extern "C" void LLVMTimeTraceProfilerFinish(const char* FileName) { |
74b04a01 XL |
74 | StringRef FN(FileName); |
75 | std::error_code EC; | |
76 | raw_fd_ostream OS(FN, EC, sys::fs::CD_CreateAlways); | |
77 | ||
78 | timeTraceProfilerWrite(OS); | |
79 | timeTraceProfilerCleanup(); | |
74b04a01 XL |
80 | } |
81 | ||
a7813a04 XL |
82 | #ifdef LLVM_COMPONENT_X86 |
83 | #define SUBTARGET_X86 SUBTARGET(X86) | |
84 | #else | |
85 | #define SUBTARGET_X86 | |
86 | #endif | |
87 | ||
88 | #ifdef LLVM_COMPONENT_ARM | |
89 | #define SUBTARGET_ARM SUBTARGET(ARM) | |
90 | #else | |
91 | #define SUBTARGET_ARM | |
92 | #endif | |
93 | ||
94 | #ifdef LLVM_COMPONENT_AARCH64 | |
95 | #define SUBTARGET_AARCH64 SUBTARGET(AArch64) | |
96 | #else | |
97 | #define SUBTARGET_AARCH64 | |
98 | #endif | |
99 | ||
f035d41b XL |
100 | #ifdef LLVM_COMPONENT_AVR |
101 | #define SUBTARGET_AVR SUBTARGET(AVR) | |
102 | #else | |
103 | #define SUBTARGET_AVR | |
104 | #endif | |
105 | ||
c295e0f8 XL |
106 | #ifdef LLVM_COMPONENT_M68k |
107 | #define SUBTARGET_M68K SUBTARGET(M68k) | |
108 | #else | |
109 | #define SUBTARGET_M68K | |
110 | #endif | |
111 | ||
add651ee FG |
112 | #ifdef LLVM_COMPONENT_CSKY |
113 | #define SUBTARGET_CSKY SUBTARGET(CSKY) | |
114 | #else | |
115 | #define SUBTARGET_CSKY | |
116 | #endif | |
117 | ||
a7813a04 XL |
118 | #ifdef LLVM_COMPONENT_MIPS |
119 | #define SUBTARGET_MIPS SUBTARGET(Mips) | |
120 | #else | |
121 | #define SUBTARGET_MIPS | |
122 | #endif | |
123 | ||
124 | #ifdef LLVM_COMPONENT_POWERPC | |
125 | #define SUBTARGET_PPC SUBTARGET(PPC) | |
126 | #else | |
127 | #define SUBTARGET_PPC | |
128 | #endif | |
129 | ||
9e0c209e SL |
130 | #ifdef LLVM_COMPONENT_SYSTEMZ |
131 | #define SUBTARGET_SYSTEMZ SUBTARGET(SystemZ) | |
132 | #else | |
133 | #define SUBTARGET_SYSTEMZ | |
134 | #endif | |
135 | ||
476ff2be SL |
136 | #ifdef LLVM_COMPONENT_MSP430 |
137 | #define SUBTARGET_MSP430 SUBTARGET(MSP430) | |
138 | #else | |
139 | #define SUBTARGET_MSP430 | |
140 | #endif | |
141 | ||
b7449926 XL |
142 | #ifdef LLVM_COMPONENT_RISCV |
143 | #define SUBTARGET_RISCV SUBTARGET(RISCV) | |
144 | #else | |
145 | #define SUBTARGET_RISCV | |
146 | #endif | |
147 | ||
32a655c1 SL |
148 | #ifdef LLVM_COMPONENT_SPARC |
149 | #define SUBTARGET_SPARC SUBTARGET(Sparc) | |
150 | #else | |
151 | #define SUBTARGET_SPARC | |
152 | #endif | |
153 | ||
7cac9316 XL |
154 | #ifdef LLVM_COMPONENT_HEXAGON |
155 | #define SUBTARGET_HEXAGON SUBTARGET(Hexagon) | |
156 | #else | |
157 | #define SUBTARGET_HEXAGON | |
158 | #endif | |
159 | ||
353b0b11 FG |
160 | #ifdef LLVM_COMPONENT_LOONGARCH |
161 | #define SUBTARGET_LOONGARCH SUBTARGET(LoongArch) | |
162 | #else | |
163 | #define SUBTARGET_LOONGARCH | |
164 | #endif | |
165 | ||
32a655c1 SL |
166 | #define GEN_SUBTARGETS \ |
167 | SUBTARGET_X86 \ | |
168 | SUBTARGET_ARM \ | |
169 | SUBTARGET_AARCH64 \ | |
f035d41b | 170 | SUBTARGET_AVR \ |
c295e0f8 | 171 | SUBTARGET_M68K \ |
add651ee | 172 | SUBTARGET_CSKY \ |
32a655c1 SL |
173 | SUBTARGET_MIPS \ |
174 | SUBTARGET_PPC \ | |
175 | SUBTARGET_SYSTEMZ \ | |
176 | SUBTARGET_MSP430 \ | |
7cac9316 | 177 | SUBTARGET_SPARC \ |
b7449926 XL |
178 | SUBTARGET_HEXAGON \ |
179 | SUBTARGET_RISCV \ | |
353b0b11 | 180 | SUBTARGET_LOONGARCH \ |
32a655c1 SL |
181 | |
182 | #define SUBTARGET(x) \ | |
183 | namespace llvm { \ | |
184 | extern const SubtargetFeatureKV x##FeatureKV[]; \ | |
185 | extern const SubtargetFeatureKV x##SubTypeKV[]; \ | |
a7813a04 XL |
186 | } |
187 | ||
188 | GEN_SUBTARGETS | |
189 | #undef SUBTARGET | |
190 | ||
32a655c1 SL |
191 | extern "C" bool LLVMRustHasFeature(LLVMTargetMachineRef TM, |
192 | const char *Feature) { | |
193 | TargetMachine *Target = unwrap(TM); | |
194 | const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); | |
0531ce1d | 195 | return MCInfo->checkFeatures(std::string("+") + Feature); |
a7813a04 XL |
196 | } |
197 | ||
5bcae85e | 198 | enum class LLVMRustCodeModel { |
f9f354fc | 199 | Tiny, |
32a655c1 SL |
200 | Small, |
201 | Kernel, | |
202 | Medium, | |
203 | Large, | |
2c00a5a8 | 204 | None, |
5bcae85e SL |
205 | }; |
206 | ||
487cf647 FG |
207 | #if LLVM_VERSION_LT(16, 0) |
208 | static Optional<CodeModel::Model> | |
209 | #else | |
210 | static std::optional<CodeModel::Model> | |
211 | #endif | |
212 | fromRust(LLVMRustCodeModel Model) { | |
32a655c1 | 213 | switch (Model) { |
f9f354fc XL |
214 | case LLVMRustCodeModel::Tiny: |
215 | return CodeModel::Tiny; | |
32a655c1 SL |
216 | case LLVMRustCodeModel::Small: |
217 | return CodeModel::Small; | |
218 | case LLVMRustCodeModel::Kernel: | |
219 | return CodeModel::Kernel; | |
220 | case LLVMRustCodeModel::Medium: | |
221 | return CodeModel::Medium; | |
222 | case LLVMRustCodeModel::Large: | |
223 | return CodeModel::Large; | |
f9f354fc | 224 | case LLVMRustCodeModel::None: |
9c376795 | 225 | #if LLVM_VERSION_LT(16, 0) |
f9f354fc | 226 | return None; |
9c376795 FG |
227 | #else |
228 | return std::nullopt; | |
229 | #endif | |
32a655c1 | 230 | default: |
ff7c6d11 | 231 | report_fatal_error("Bad CodeModel."); |
5bcae85e SL |
232 | } |
233 | } | |
234 | ||
235 | enum class LLVMRustCodeGenOptLevel { | |
32a655c1 SL |
236 | None, |
237 | Less, | |
238 | Default, | |
239 | Aggressive, | |
5bcae85e SL |
240 | }; |
241 | ||
781aab86 FG |
242 | #if LLVM_VERSION_GE(18, 0) |
243 | using CodeGenOptLevelEnum = llvm::CodeGenOptLevel; | |
244 | #else | |
245 | using CodeGenOptLevelEnum = llvm::CodeGenOpt::Level; | |
246 | #endif | |
247 | ||
248 | static CodeGenOptLevelEnum fromRust(LLVMRustCodeGenOptLevel Level) { | |
32a655c1 SL |
249 | switch (Level) { |
250 | case LLVMRustCodeGenOptLevel::None: | |
781aab86 | 251 | return CodeGenOptLevelEnum::None; |
32a655c1 | 252 | case LLVMRustCodeGenOptLevel::Less: |
781aab86 | 253 | return CodeGenOptLevelEnum::Less; |
32a655c1 | 254 | case LLVMRustCodeGenOptLevel::Default: |
781aab86 | 255 | return CodeGenOptLevelEnum::Default; |
32a655c1 | 256 | case LLVMRustCodeGenOptLevel::Aggressive: |
781aab86 | 257 | return CodeGenOptLevelEnum::Aggressive; |
32a655c1 | 258 | default: |
ff7c6d11 | 259 | report_fatal_error("Bad CodeGenOptLevel."); |
5bcae85e SL |
260 | } |
261 | } | |
262 | ||
74b04a01 XL |
263 | enum class LLVMRustPassBuilderOptLevel { |
264 | O0, | |
265 | O1, | |
266 | O2, | |
267 | O3, | |
268 | Os, | |
269 | Oz, | |
270 | }; | |
271 | ||
94222f64 | 272 | static OptimizationLevel fromRust(LLVMRustPassBuilderOptLevel Level) { |
74b04a01 XL |
273 | switch (Level) { |
274 | case LLVMRustPassBuilderOptLevel::O0: | |
94222f64 | 275 | return OptimizationLevel::O0; |
74b04a01 | 276 | case LLVMRustPassBuilderOptLevel::O1: |
94222f64 | 277 | return OptimizationLevel::O1; |
74b04a01 | 278 | case LLVMRustPassBuilderOptLevel::O2: |
94222f64 | 279 | return OptimizationLevel::O2; |
74b04a01 | 280 | case LLVMRustPassBuilderOptLevel::O3: |
94222f64 | 281 | return OptimizationLevel::O3; |
74b04a01 | 282 | case LLVMRustPassBuilderOptLevel::Os: |
94222f64 | 283 | return OptimizationLevel::Os; |
74b04a01 | 284 | case LLVMRustPassBuilderOptLevel::Oz: |
94222f64 | 285 | return OptimizationLevel::Oz; |
74b04a01 XL |
286 | default: |
287 | report_fatal_error("Bad PassBuilderOptLevel."); | |
288 | } | |
289 | } | |
290 | ||
f9f354fc | 291 | enum class LLVMRustRelocModel { |
7cac9316 XL |
292 | Static, |
293 | PIC, | |
294 | DynamicNoPic, | |
295 | ROPI, | |
296 | RWPI, | |
297 | ROPIRWPI, | |
298 | }; | |
299 | ||
f9f354fc | 300 | static Reloc::Model fromRust(LLVMRustRelocModel RustReloc) { |
7cac9316 | 301 | switch (RustReloc) { |
f9f354fc | 302 | case LLVMRustRelocModel::Static: |
7cac9316 | 303 | return Reloc::Static; |
f9f354fc | 304 | case LLVMRustRelocModel::PIC: |
7cac9316 | 305 | return Reloc::PIC_; |
f9f354fc | 306 | case LLVMRustRelocModel::DynamicNoPic: |
7cac9316 | 307 | return Reloc::DynamicNoPIC; |
f9f354fc | 308 | case LLVMRustRelocModel::ROPI: |
7cac9316 | 309 | return Reloc::ROPI; |
f9f354fc | 310 | case LLVMRustRelocModel::RWPI: |
7cac9316 | 311 | return Reloc::RWPI; |
f9f354fc | 312 | case LLVMRustRelocModel::ROPIRWPI: |
7cac9316 | 313 | return Reloc::ROPI_RWPI; |
7cac9316 | 314 | } |
ff7c6d11 | 315 | report_fatal_error("Bad RelocModel."); |
7cac9316 XL |
316 | } |
317 | ||
5bcae85e | 318 | /// getLongestEntryLength - Return the length of the longest entry in the table. |
416331ca XL |
319 | template<typename KV> |
320 | static size_t getLongestEntryLength(ArrayRef<KV> Table) { | |
5bcae85e SL |
321 | size_t MaxLen = 0; |
322 | for (auto &I : Table) | |
323 | MaxLen = std::max(MaxLen, std::strlen(I.Key)); | |
324 | return MaxLen; | |
325 | } | |
326 | ||
add651ee FG |
327 | using PrintBackendInfo = void(void*, const char* Data, size_t Len); |
328 | ||
329 | extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef TM, | |
330 | const char* TargetCPU, | |
331 | PrintBackendInfo Print, | |
332 | void* Out) { | |
32a655c1 | 333 | const TargetMachine *Target = unwrap(TM); |
49aad941 | 334 | const Triple::ArchType HostArch = Triple(sys::getDefaultTargetTriple()).getArch(); |
ea8adc8c | 335 | const Triple::ArchType TargetArch = Target->getTargetTriple().getArch(); |
49aad941 | 336 | |
add651ee FG |
337 | std::ostringstream Buf; |
338 | ||
49aad941 | 339 | #if LLVM_VERSION_GE(17, 0) |
781aab86 | 340 | const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); |
49aad941 | 341 | const ArrayRef<SubtargetSubTypeKV> CPUTable = MCInfo->getAllProcessorDescriptions(); |
49aad941 | 342 | #else |
add651ee | 343 | Buf << "Full target CPU help is not supported by this LLVM version.\n\n"; |
49aad941 FG |
344 | SubtargetSubTypeKV TargetCPUKV = { TargetCPU, {{}}, {{}} }; |
345 | const ArrayRef<SubtargetSubTypeKV> CPUTable = TargetCPUKV; | |
346 | #endif | |
32a655c1 SL |
347 | unsigned MaxCPULen = getLongestEntryLength(CPUTable); |
348 | ||
add651ee | 349 | Buf << "Available CPUs for this target:\n"; |
49aad941 FG |
350 | // Don't print the "native" entry when the user specifies --target with a |
351 | // different arch since that could be wrong or misleading. | |
ea8adc8c | 352 | if (HostArch == TargetArch) { |
49aad941 | 353 | MaxCPULen = std::max(MaxCPULen, (unsigned) std::strlen("native")); |
ea8adc8c | 354 | const StringRef HostCPU = sys::getHostCPUName(); |
add651ee FG |
355 | Buf << " " << std::left << std::setw(MaxCPULen) << "native" |
356 | << " - Select the CPU of the current host " | |
357 | "(currently " << HostCPU.str() << ").\n"; | |
ea8adc8c | 358 | } |
49aad941 FG |
359 | for (auto &CPU : CPUTable) { |
360 | // Compare cpu against current target to label the default | |
361 | if (strcmp(CPU.Key, TargetCPU) == 0) { | |
add651ee FG |
362 | Buf << " " << std::left << std::setw(MaxCPULen) << CPU.Key |
363 | << " - This is the default target CPU for the current build target " | |
364 | "(currently " << Target->getTargetTriple().str() << ")."; | |
49aad941 FG |
365 | } |
366 | else { | |
add651ee | 367 | Buf << " " << CPU.Key; |
49aad941 | 368 | } |
add651ee | 369 | Buf << "\n"; |
49aad941 | 370 | } |
add651ee FG |
371 | |
372 | const auto &BufString = Buf.str(); | |
373 | Print(Out, BufString.data(), BufString.size()); | |
5bcae85e SL |
374 | } |
375 | ||
cdc7bbd5 | 376 | extern "C" size_t LLVMRustGetTargetFeaturesCount(LLVMTargetMachineRef TM) { |
49aad941 | 377 | #ifdef LLVM_RUSTLLVM |
32a655c1 SL |
378 | const TargetMachine *Target = unwrap(TM); |
379 | const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); | |
380 | const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getFeatureTable(); | |
cdc7bbd5 | 381 | return FeatTable.size(); |
49aad941 FG |
382 | #else |
383 | return 0; | |
384 | #endif | |
cdc7bbd5 | 385 | } |
32a655c1 | 386 | |
cdc7bbd5 XL |
387 | extern "C" void LLVMRustGetTargetFeature(LLVMTargetMachineRef TM, size_t Index, |
388 | const char** Feature, const char** Desc) { | |
49aad941 | 389 | #ifdef LLVM_RUSTLLVM |
cdc7bbd5 XL |
390 | const TargetMachine *Target = unwrap(TM); |
391 | const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); | |
392 | const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getFeatureTable(); | |
393 | const SubtargetFeatureKV Feat = FeatTable[Index]; | |
394 | *Feature = Feat.Key; | |
395 | *Desc = Feat.Desc; | |
5bcae85e | 396 | #endif |
49aad941 | 397 | } |
5bcae85e | 398 | |
b7449926 XL |
399 | extern "C" const char* LLVMRustGetHostCPUName(size_t *len) { |
400 | StringRef Name = sys::getHostCPUName(); | |
401 | *len = Name.size(); | |
402 | return Name.data(); | |
403 | } | |
404 | ||
32a655c1 SL |
405 | extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine( |
406 | const char *TripleStr, const char *CPU, const char *Feature, | |
f9f354fc | 407 | const char *ABIStr, LLVMRustCodeModel RustCM, LLVMRustRelocModel RustReloc, |
32a655c1 | 408 | LLVMRustCodeGenOptLevel RustOptLevel, bool UseSoftFloat, |
f9f354fc | 409 | bool FunctionSections, |
abe05a73 | 410 | bool DataSections, |
3c0e092e | 411 | bool UniqueSectionNames, |
abe05a73 | 412 | bool TrapUnreachable, |
b7449926 | 413 | bool Singlethread, |
0bf4aa26 | 414 | bool AsmComments, |
60c5eb7d | 415 | bool EmitStackSizeSection, |
f9f354fc | 416 | bool RelaxELFRelocations, |
fc512014 | 417 | bool UseInitArray, |
353b0b11 | 418 | const char *SplitDwarfFile, |
781aab86 FG |
419 | const char *OutputObjFile, |
420 | const char *DebugInfoCompression, | |
421 | bool ForceEmulatedTls, | |
422 | const char *ArgsCstrBuff, size_t ArgsCstrBuffLen) { | |
5bcae85e | 423 | |
32a655c1 | 424 | auto OptLevel = fromRust(RustOptLevel); |
7cac9316 | 425 | auto RM = fromRust(RustReloc); |
f9f354fc | 426 | auto CM = fromRust(RustCM); |
5bcae85e | 427 | |
32a655c1 SL |
428 | std::string Error; |
429 | Triple Trip(Triple::normalize(TripleStr)); | |
430 | const llvm::Target *TheTarget = | |
431 | TargetRegistry::lookupTarget(Trip.getTriple(), Error); | |
432 | if (TheTarget == nullptr) { | |
433 | LLVMRustSetLastError(Error.c_str()); | |
434 | return nullptr; | |
435 | } | |
1a4d82fc | 436 | |
781aab86 | 437 | TargetOptions Options = codegen::InitTargetOptionsFromCodeGenFlags(Trip); |
5bcae85e | 438 | |
32a655c1 SL |
439 | Options.FloatABIType = FloatABI::Default; |
440 | if (UseSoftFloat) { | |
441 | Options.FloatABIType = FloatABI::Soft; | |
442 | } | |
443 | Options.DataSections = DataSections; | |
444 | Options.FunctionSections = FunctionSections; | |
3c0e092e | 445 | Options.UniqueSectionNames = UniqueSectionNames; |
b7449926 XL |
446 | Options.MCOptions.AsmVerbose = AsmComments; |
447 | Options.MCOptions.PreserveAsmComments = AsmComments; | |
60c5eb7d | 448 | Options.MCOptions.ABIName = ABIStr; |
fc512014 XL |
449 | if (SplitDwarfFile) { |
450 | Options.MCOptions.SplitDwarfFile = SplitDwarfFile; | |
451 | } | |
781aab86 FG |
452 | if (OutputObjFile) { |
453 | Options.ObjectFilenameForDebug = OutputObjFile; | |
454 | } | |
455 | #if LLVM_VERSION_GE(16, 0) | |
456 | if (!strcmp("zlib", DebugInfoCompression) && llvm::compression::zlib::isAvailable()) { | |
457 | Options.CompressDebugSections = DebugCompressionType::Zlib; | |
458 | } else if (!strcmp("zstd", DebugInfoCompression) && llvm::compression::zstd::isAvailable()) { | |
459 | Options.CompressDebugSections = DebugCompressionType::Zstd; | |
460 | } else if (!strcmp("none", DebugInfoCompression)) { | |
461 | Options.CompressDebugSections = DebugCompressionType::None; | |
462 | } | |
463 | #endif | |
464 | ||
60c5eb7d | 465 | Options.RelaxELFRelocations = RelaxELFRelocations; |
f9f354fc | 466 | Options.UseInitArray = UseInitArray; |
49aad941 FG |
467 | |
468 | #if LLVM_VERSION_LT(17, 0) | |
353b0b11 FG |
469 | if (ForceEmulatedTls) { |
470 | Options.ExplicitEmulatedTLS = true; | |
471 | Options.EmulatedTLS = true; | |
472 | } | |
49aad941 FG |
473 | #else |
474 | Options.EmulatedTLS = ForceEmulatedTls || Trip.hasDefaultEmulatedTLS(); | |
475 | #endif | |
32a655c1 | 476 | |
abe05a73 | 477 | if (TrapUnreachable) { |
94b46f34 | 478 | // Tell LLVM to codegen `unreachable` into an explicit trap instruction. |
abe05a73 XL |
479 | // This limits the extent of possible undefined behavior in some cases, as |
480 | // it prevents control flow from "falling through" into whatever code | |
481 | // happens to be laid out next in memory. | |
482 | Options.TrapUnreachable = true; | |
483 | } | |
484 | ||
485 | if (Singlethread) { | |
486 | Options.ThreadModel = ThreadModel::Single; | |
487 | } | |
488 | ||
0bf4aa26 XL |
489 | Options.EmitStackSizeSection = EmitStackSizeSection; |
490 | ||
781aab86 FG |
491 | |
492 | if (ArgsCstrBuff != nullptr) | |
493 | { | |
494 | int buffer_offset = 0; | |
495 | assert(ArgsCstrBuff[ArgsCstrBuffLen - 1] == '\0'); | |
496 | ||
497 | const size_t arg0_len = std::strlen(ArgsCstrBuff); | |
498 | char* arg0 = new char[arg0_len + 1]; | |
499 | memcpy(arg0, ArgsCstrBuff, arg0_len); | |
500 | arg0[arg0_len] = '\0'; | |
501 | buffer_offset += arg0_len + 1; | |
502 | ||
503 | const int num_cmd_arg_strings = | |
504 | std::count(&ArgsCstrBuff[buffer_offset], &ArgsCstrBuff[ArgsCstrBuffLen], '\0'); | |
505 | ||
506 | std::string* cmd_arg_strings = new std::string[num_cmd_arg_strings]; | |
507 | for (int i = 0; i < num_cmd_arg_strings; ++i) | |
508 | { | |
509 | assert(buffer_offset < ArgsCstrBuffLen); | |
510 | const int len = std::strlen(ArgsCstrBuff + buffer_offset); | |
511 | cmd_arg_strings[i] = std::string(&ArgsCstrBuff[buffer_offset], len); | |
512 | buffer_offset += len + 1; | |
513 | } | |
514 | ||
515 | assert(buffer_offset == ArgsCstrBuffLen); | |
516 | ||
517 | Options.MCOptions.Argv0 = arg0; | |
518 | Options.MCOptions.CommandLineArgs = | |
519 | llvm::ArrayRef<std::string>(cmd_arg_strings, num_cmd_arg_strings); | |
520 | } | |
521 | ||
32a655c1 | 522 | TargetMachine *TM = TheTarget->createTargetMachine( |
b7449926 | 523 | Trip.getTriple(), CPU, Feature, Options, RM, CM, OptLevel); |
32a655c1 | 524 | return wrap(TM); |
1a4d82fc JJ |
525 | } |
526 | ||
32a655c1 | 527 | extern "C" void LLVMRustDisposeTargetMachine(LLVMTargetMachineRef TM) { |
781aab86 FG |
528 | |
529 | MCTargetOptions& MCOptions = unwrap(TM)->Options.MCOptions; | |
530 | delete[] MCOptions.Argv0; | |
531 | delete[] MCOptions.CommandLineArgs.data(); | |
532 | ||
32a655c1 | 533 | delete unwrap(TM); |
1a4d82fc JJ |
534 | } |
535 | ||
1a4d82fc JJ |
536 | // Unfortunately, the LLVM C API doesn't provide a way to create the |
537 | // TargetLibraryInfo pass, so we use this method to do so. | |
32a655c1 SL |
538 | extern "C" void LLVMRustAddLibraryInfo(LLVMPassManagerRef PMR, LLVMModuleRef M, |
539 | bool DisableSimplifyLibCalls) { | |
540 | Triple TargetTriple(unwrap(M)->getTargetTriple()); | |
541 | TargetLibraryInfoImpl TLII(TargetTriple); | |
542 | if (DisableSimplifyLibCalls) | |
543 | TLII.disableAllFunctions(); | |
544 | unwrap(PMR)->add(new TargetLibraryInfoWrapperPass(TLII)); | |
1a4d82fc JJ |
545 | } |
546 | ||
32a655c1 SL |
547 | extern "C" void LLVMRustSetLLVMOptions(int Argc, char **Argv) { |
548 | // Initializing the command-line options more than once is not allowed. So, | |
9c376795 | 549 | // check if they've already been initialized. (This could happen if we're |
32a655c1 SL |
550 | // being called from rustpkg, for example). If the arguments change, then |
551 | // that's just kinda unfortunate. | |
552 | static bool Initialized = false; | |
553 | if (Initialized) | |
554 | return; | |
555 | Initialized = true; | |
556 | cl::ParseCommandLineOptions(Argc, Argv); | |
1a4d82fc JJ |
557 | } |
558 | ||
5bcae85e | 559 | enum class LLVMRustFileType { |
32a655c1 SL |
560 | AssemblyFile, |
561 | ObjectFile, | |
5bcae85e SL |
562 | }; |
563 | ||
dfeec247 XL |
564 | static CodeGenFileType fromRust(LLVMRustFileType Type) { |
565 | switch (Type) { | |
566 | case LLVMRustFileType::AssemblyFile: | |
781aab86 FG |
567 | #if LLVM_VERSION_GE(18, 0) |
568 | return CodeGenFileType::AssemblyFile; | |
569 | #else | |
dfeec247 | 570 | return CGFT_AssemblyFile; |
781aab86 | 571 | #endif |
dfeec247 | 572 | case LLVMRustFileType::ObjectFile: |
781aab86 FG |
573 | #if LLVM_VERSION_GE(18, 0) |
574 | return CodeGenFileType::ObjectFile; | |
575 | #else | |
dfeec247 | 576 | return CGFT_ObjectFile; |
781aab86 | 577 | #endif |
dfeec247 XL |
578 | default: |
579 | report_fatal_error("Bad FileType."); | |
580 | } | |
581 | } | |
5bcae85e SL |
582 | |
583 | extern "C" LLVMRustResult | |
32a655c1 | 584 | LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR, |
fc512014 | 585 | LLVMModuleRef M, const char *Path, const char *DwoPath, |
32a655c1 | 586 | LLVMRustFileType RustFileType) { |
5bcae85e | 587 | llvm::legacy::PassManager *PM = unwrap<llvm::legacy::PassManager>(PMR); |
32a655c1 | 588 | auto FileType = fromRust(RustFileType); |
1a4d82fc JJ |
589 | |
590 | std::string ErrorInfo; | |
1a4d82fc | 591 | std::error_code EC; |
17df50a5 | 592 | raw_fd_ostream OS(Path, EC, sys::fs::OF_None); |
1a4d82fc JJ |
593 | if (EC) |
594 | ErrorInfo = EC.message(); | |
1a4d82fc JJ |
595 | if (ErrorInfo != "") { |
596 | LLVMRustSetLastError(ErrorInfo.c_str()); | |
5bcae85e | 597 | return LLVMRustResult::Failure; |
1a4d82fc | 598 | } |
1a4d82fc | 599 | |
b7449926 | 600 | buffer_ostream BOS(OS); |
fc512014 | 601 | if (DwoPath) { |
17df50a5 | 602 | raw_fd_ostream DOS(DwoPath, EC, sys::fs::OF_None); |
fc512014 XL |
603 | EC.clear(); |
604 | if (EC) | |
605 | ErrorInfo = EC.message(); | |
606 | if (ErrorInfo != "") { | |
607 | LLVMRustSetLastError(ErrorInfo.c_str()); | |
608 | return LLVMRustResult::Failure; | |
609 | } | |
610 | buffer_ostream DBOS(DOS); | |
611 | unwrap(Target)->addPassesToEmitFile(*PM, BOS, &DBOS, FileType, false); | |
612 | PM->run(*unwrap(M)); | |
613 | } else { | |
614 | unwrap(Target)->addPassesToEmitFile(*PM, BOS, nullptr, FileType, false); | |
615 | PM->run(*unwrap(M)); | |
616 | } | |
62682a34 | 617 | |
b039eaaf | 618 | // Apparently `addPassesToEmitFile` adds a pointer to our on-the-stack output |
62682a34 SL |
619 | // stream (OS), so the only real safe place to delete this is here? Don't we |
620 | // wish this was written in Rust? | |
60c5eb7d | 621 | LLVMDisposePassManager(PMR); |
5bcae85e | 622 | return LLVMRustResult::Success; |
1a4d82fc JJ |
623 | } |
624 | ||
74b04a01 XL |
625 | extern "C" typedef void (*LLVMRustSelfProfileBeforePassCallback)(void*, // LlvmSelfProfiler |
626 | const char*, // pass name | |
627 | const char*); // IR name | |
628 | extern "C" typedef void (*LLVMRustSelfProfileAfterPassCallback)(void*); // LlvmSelfProfiler | |
629 | ||
74b04a01 | 630 | std::string LLVMRustwrappedIrGetName(const llvm::Any &WrappedIr) { |
49aad941 FG |
631 | if (const auto *Cast = any_cast<const Module *>(&WrappedIr)) |
632 | return (*Cast)->getName().str(); | |
633 | if (const auto *Cast = any_cast<const Function *>(&WrappedIr)) | |
634 | return (*Cast)->getName().str(); | |
635 | if (const auto *Cast = any_cast<const Loop *>(&WrappedIr)) | |
636 | return (*Cast)->getName().str(); | |
637 | if (const auto *Cast = any_cast<const LazyCallGraph::SCC *>(&WrappedIr)) | |
638 | return (*Cast)->getName(); | |
74b04a01 XL |
639 | return "<UNKNOWN>"; |
640 | } | |
641 | ||
642 | ||
643 | void LLVMSelfProfileInitializeCallbacks( | |
644 | PassInstrumentationCallbacks& PIC, void* LlvmSelfProfiler, | |
645 | LLVMRustSelfProfileBeforePassCallback BeforePassCallback, | |
646 | LLVMRustSelfProfileAfterPassCallback AfterPassCallback) { | |
6a06907d XL |
647 | PIC.registerBeforeNonSkippedPassCallback([LlvmSelfProfiler, BeforePassCallback]( |
648 | StringRef Pass, llvm::Any Ir) { | |
649 | std::string PassName = Pass.str(); | |
650 | std::string IrName = LLVMRustwrappedIrGetName(Ir); | |
651 | BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str()); | |
652 | }); | |
653 | ||
654 | PIC.registerAfterPassCallback( | |
655 | [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any IR, | |
656 | const PreservedAnalyses &Preserved) { | |
657 | AfterPassCallback(LlvmSelfProfiler); | |
658 | }); | |
659 | ||
660 | PIC.registerAfterPassInvalidatedCallback( | |
661 | [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, const PreservedAnalyses &Preserved) { | |
662 | AfterPassCallback(LlvmSelfProfiler); | |
663 | }); | |
74b04a01 XL |
664 | |
665 | PIC.registerBeforeAnalysisCallback([LlvmSelfProfiler, BeforePassCallback]( | |
666 | StringRef Pass, llvm::Any Ir) { | |
667 | std::string PassName = Pass.str(); | |
668 | std::string IrName = LLVMRustwrappedIrGetName(Ir); | |
669 | BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str()); | |
670 | }); | |
671 | ||
672 | PIC.registerAfterAnalysisCallback( | |
673 | [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any Ir) { | |
674 | AfterPassCallback(LlvmSelfProfiler); | |
675 | }); | |
676 | } | |
74b04a01 XL |
677 | |
678 | enum class LLVMRustOptStage { | |
679 | PreLinkNoLTO, | |
680 | PreLinkThinLTO, | |
681 | PreLinkFatLTO, | |
682 | ThinLTO, | |
683 | FatLTO, | |
684 | }; | |
685 | ||
686 | struct LLVMRustSanitizerOptions { | |
f035d41b XL |
687 | bool SanitizeAddress; |
688 | bool SanitizeAddressRecover; | |
add651ee FG |
689 | bool SanitizeCFI; |
690 | bool SanitizeKCFI; | |
74b04a01 | 691 | bool SanitizeMemory; |
f035d41b XL |
692 | bool SanitizeMemoryRecover; |
693 | int SanitizeMemoryTrackOrigins; | |
74b04a01 | 694 | bool SanitizeThread; |
6a06907d XL |
695 | bool SanitizeHWAddress; |
696 | bool SanitizeHWAddressRecover; | |
9ffffee4 FG |
697 | bool SanitizeKernelAddress; |
698 | bool SanitizeKernelAddressRecover; | |
74b04a01 XL |
699 | }; |
700 | ||
17df50a5 | 701 | extern "C" LLVMRustResult |
2b03887a | 702 | LLVMRustOptimize( |
74b04a01 XL |
703 | LLVMModuleRef ModuleRef, |
704 | LLVMTargetMachineRef TMRef, | |
705 | LLVMRustPassBuilderOptLevel OptLevelRust, | |
706 | LLVMRustOptStage OptStage, | |
add651ee | 707 | bool IsLinkerPluginLTO, |
74b04a01 XL |
708 | bool NoPrepopulatePasses, bool VerifyIR, bool UseThinLTOBuffers, |
709 | bool MergeFunctions, bool UnrollLoops, bool SLPVectorize, bool LoopVectorize, | |
f9f354fc | 710 | bool DisableSimplifyLibCalls, bool EmitLifetimeMarkers, |
74b04a01 XL |
711 | LLVMRustSanitizerOptions *SanitizerOptions, |
712 | const char *PGOGenPath, const char *PGOUsePath, | |
f2b60f7d FG |
713 | bool InstrumentCoverage, const char *InstrProfileOutput, |
714 | bool InstrumentGCOV, | |
c295e0f8 | 715 | const char *PGOSampleUsePath, bool DebugInfoForProfiling, |
74b04a01 XL |
716 | void* LlvmSelfProfiler, |
717 | LLVMRustSelfProfileBeforePassCallback BeforePassCallback, | |
17df50a5 | 718 | LLVMRustSelfProfileAfterPassCallback AfterPassCallback, |
a2a8927a XL |
719 | const char *ExtraPasses, size_t ExtraPassesLen, |
720 | const char *LLVMPlugins, size_t LLVMPluginsLen) { | |
74b04a01 XL |
721 | Module *TheModule = unwrap(ModuleRef); |
722 | TargetMachine *TM = unwrap(TMRef); | |
94222f64 | 723 | OptimizationLevel OptLevel = fromRust(OptLevelRust); |
74b04a01 | 724 | |
74b04a01 XL |
725 | |
726 | PipelineTuningOptions PTO; | |
727 | PTO.LoopUnrolling = UnrollLoops; | |
728 | PTO.LoopInterleaving = UnrollLoops; | |
729 | PTO.LoopVectorization = LoopVectorize; | |
730 | PTO.SLPVectorization = SLPVectorize; | |
6a06907d | 731 | PTO.MergeFunctions = MergeFunctions; |
6a06907d XL |
732 | |
733 | // FIXME: We may want to expose this as an option. | |
734 | bool DebugPassManager = false; | |
74b04a01 XL |
735 | |
736 | PassInstrumentationCallbacks PIC; | |
487cf647 | 737 | #if LLVM_VERSION_LT(16, 0) |
6a06907d | 738 | StandardInstrumentations SI(DebugPassManager); |
487cf647 FG |
739 | #else |
740 | StandardInstrumentations SI(TheModule->getContext(), DebugPassManager); | |
741 | #endif | |
74b04a01 XL |
742 | SI.registerCallbacks(PIC); |
743 | ||
744 | if (LlvmSelfProfiler){ | |
745 | LLVMSelfProfileInitializeCallbacks(PIC,LlvmSelfProfiler,BeforePassCallback,AfterPassCallback); | |
746 | } | |
747 | ||
487cf647 | 748 | #if LLVM_VERSION_LT(16, 0) |
74b04a01 | 749 | Optional<PGOOptions> PGOOpt; |
487cf647 FG |
750 | #else |
751 | std::optional<PGOOptions> PGOOpt; | |
9ffffee4 FG |
752 | #endif |
753 | #if LLVM_VERSION_GE(17, 0) | |
754 | auto FS = vfs::getRealFileSystem(); | |
487cf647 | 755 | #endif |
74b04a01 | 756 | if (PGOGenPath) { |
c295e0f8 | 757 | assert(!PGOUsePath && !PGOSampleUsePath); |
9ffffee4 FG |
758 | PGOOpt = PGOOptions(PGOGenPath, "", "", |
759 | #if LLVM_VERSION_GE(17, 0) | |
add651ee | 760 | "", |
9ffffee4 FG |
761 | FS, |
762 | #endif | |
763 | PGOOptions::IRInstr, PGOOptions::NoCSAction, | |
764 | DebugInfoForProfiling); | |
74b04a01 | 765 | } else if (PGOUsePath) { |
c295e0f8 | 766 | assert(!PGOSampleUsePath); |
9ffffee4 FG |
767 | PGOOpt = PGOOptions(PGOUsePath, "", "", |
768 | #if LLVM_VERSION_GE(17, 0) | |
add651ee | 769 | "", |
9ffffee4 FG |
770 | FS, |
771 | #endif | |
772 | PGOOptions::IRUse, PGOOptions::NoCSAction, | |
773 | DebugInfoForProfiling); | |
c295e0f8 | 774 | } else if (PGOSampleUsePath) { |
9ffffee4 FG |
775 | PGOOpt = PGOOptions(PGOSampleUsePath, "", "", |
776 | #if LLVM_VERSION_GE(17, 0) | |
add651ee | 777 | "", |
9ffffee4 FG |
778 | FS, |
779 | #endif | |
780 | PGOOptions::SampleUse, PGOOptions::NoCSAction, | |
781 | DebugInfoForProfiling); | |
c295e0f8 | 782 | } else if (DebugInfoForProfiling) { |
9ffffee4 FG |
783 | PGOOpt = PGOOptions("", "", "", |
784 | #if LLVM_VERSION_GE(17, 0) | |
add651ee | 785 | "", |
9ffffee4 FG |
786 | FS, |
787 | #endif | |
788 | PGOOptions::NoAction, PGOOptions::NoCSAction, | |
789 | DebugInfoForProfiling); | |
74b04a01 XL |
790 | } |
791 | ||
3c0e092e | 792 | PassBuilder PB(TM, PTO, PGOOpt, &PIC); |
17df50a5 XL |
793 | LoopAnalysisManager LAM; |
794 | FunctionAnalysisManager FAM; | |
795 | CGSCCAnalysisManager CGAM; | |
796 | ModuleAnalysisManager MAM; | |
74b04a01 XL |
797 | |
798 | FAM.registerPass([&] { return PB.buildDefaultAAPipeline(); }); | |
799 | ||
800 | Triple TargetTriple(TheModule->getTargetTriple()); | |
801 | std::unique_ptr<TargetLibraryInfoImpl> TLII(new TargetLibraryInfoImpl(TargetTriple)); | |
802 | if (DisableSimplifyLibCalls) | |
803 | TLII->disableAllFunctions(); | |
804 | FAM.registerPass([&] { return TargetLibraryAnalysis(*TLII); }); | |
805 | ||
806 | PB.registerModuleAnalyses(MAM); | |
807 | PB.registerCGSCCAnalyses(CGAM); | |
808 | PB.registerFunctionAnalyses(FAM); | |
809 | PB.registerLoopAnalyses(LAM); | |
810 | PB.crossRegisterProxies(LAM, FAM, CGAM, MAM); | |
811 | ||
812 | // We manually collect pipeline callbacks so we can apply them at O0, where the | |
813 | // PassBuilder does not create a pipeline. | |
94222f64 | 814 | std::vector<std::function<void(ModulePassManager &, OptimizationLevel)>> |
6a06907d | 815 | PipelineStartEPCallbacks; |
94222f64 | 816 | std::vector<std::function<void(ModulePassManager &, OptimizationLevel)>> |
f035d41b | 817 | OptimizerLastEPCallbacks; |
74b04a01 | 818 | |
add651ee FG |
819 | if (!IsLinkerPluginLTO |
820 | && SanitizerOptions && SanitizerOptions->SanitizeCFI | |
821 | && !NoPrepopulatePasses) { | |
822 | PipelineStartEPCallbacks.push_back( | |
823 | [](ModulePassManager &MPM, OptimizationLevel Level) { | |
824 | MPM.addPass(LowerTypeTestsPass(/*ExportSummary=*/nullptr, | |
825 | /*ImportSummary=*/nullptr, | |
826 | /*DropTypeTests=*/false)); | |
827 | } | |
828 | ); | |
829 | } | |
830 | ||
74b04a01 | 831 | if (VerifyIR) { |
6a06907d | 832 | PipelineStartEPCallbacks.push_back( |
94222f64 | 833 | [VerifyIR](ModulePassManager &MPM, OptimizationLevel Level) { |
74b04a01 | 834 | MPM.addPass(VerifierPass()); |
6a06907d XL |
835 | } |
836 | ); | |
74b04a01 XL |
837 | } |
838 | ||
17df50a5 XL |
839 | if (InstrumentGCOV) { |
840 | PipelineStartEPCallbacks.push_back( | |
94222f64 | 841 | [](ModulePassManager &MPM, OptimizationLevel Level) { |
17df50a5 XL |
842 | MPM.addPass(GCOVProfilerPass(GCOVOptions::getDefault())); |
843 | } | |
844 | ); | |
845 | } | |
846 | ||
847 | if (InstrumentCoverage) { | |
848 | PipelineStartEPCallbacks.push_back( | |
f2b60f7d | 849 | [InstrProfileOutput](ModulePassManager &MPM, OptimizationLevel Level) { |
17df50a5 | 850 | InstrProfOptions Options; |
f2b60f7d FG |
851 | if (InstrProfileOutput) { |
852 | Options.InstrProfileOutput = InstrProfileOutput; | |
853 | } | |
49aad941 FG |
854 | // cargo run tests in multhreading mode by default |
855 | // so use atomics for coverage counters | |
856 | Options.Atomic = true; | |
17df50a5 XL |
857 | MPM.addPass(InstrProfiling(Options, false)); |
858 | } | |
859 | ); | |
860 | } | |
861 | ||
74b04a01 XL |
862 | if (SanitizerOptions) { |
863 | if (SanitizerOptions->SanitizeMemory) { | |
f2b60f7d FG |
864 | MemorySanitizerOptions Options( |
865 | SanitizerOptions->SanitizeMemoryTrackOrigins, | |
866 | SanitizerOptions->SanitizeMemoryRecover, | |
867 | /*CompileKernel=*/false, | |
868 | /*EagerChecks=*/true); | |
f035d41b | 869 | OptimizerLastEPCallbacks.push_back( |
94222f64 | 870 | [Options](ModulePassManager &MPM, OptimizationLevel Level) { |
9ffffee4 | 871 | #if LLVM_VERSION_LT(16, 0) |
c295e0f8 | 872 | MPM.addPass(ModuleMemorySanitizerPass(Options)); |
9ffffee4 | 873 | MPM.addPass(createModuleToFunctionPassAdaptor(MemorySanitizerPass(Options))); |
c295e0f8 | 874 | #else |
f035d41b | 875 | MPM.addPass(MemorySanitizerPass(Options)); |
f2b60f7d | 876 | #endif |
f035d41b XL |
877 | } |
878 | ); | |
74b04a01 XL |
879 | } |
880 | ||
881 | if (SanitizerOptions->SanitizeThread) { | |
f035d41b | 882 | OptimizerLastEPCallbacks.push_back( |
94222f64 | 883 | [](ModulePassManager &MPM, OptimizationLevel Level) { |
c295e0f8 | 884 | MPM.addPass(ModuleThreadSanitizerPass()); |
f035d41b XL |
885 | MPM.addPass(createModuleToFunctionPassAdaptor(ThreadSanitizerPass())); |
886 | } | |
887 | ); | |
74b04a01 XL |
888 | } |
889 | ||
9ffffee4 | 890 | if (SanitizerOptions->SanitizeAddress || SanitizerOptions->SanitizeKernelAddress) { |
f035d41b | 891 | OptimizerLastEPCallbacks.push_back( |
94222f64 | 892 | [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) { |
9ffffee4 | 893 | auto CompileKernel = SanitizerOptions->SanitizeKernelAddress; |
3c0e092e | 894 | AddressSanitizerOptions opts = AddressSanitizerOptions{ |
9ffffee4 FG |
895 | CompileKernel, |
896 | SanitizerOptions->SanitizeAddressRecover | |
897 | || SanitizerOptions->SanitizeKernelAddressRecover, | |
3c0e092e XL |
898 | /*UseAfterScope=*/true, |
899 | AsanDetectStackUseAfterReturnMode::Runtime, | |
900 | }; | |
f2b60f7d | 901 | #if LLVM_VERSION_LT(16, 0) |
3c0e092e | 902 | MPM.addPass(ModuleAddressSanitizerPass(opts)); |
f2b60f7d FG |
903 | #else |
904 | MPM.addPass(AddressSanitizerPass(opts)); | |
94222f64 | 905 | #endif |
f035d41b XL |
906 | } |
907 | ); | |
6a06907d XL |
908 | } |
909 | if (SanitizerOptions->SanitizeHWAddress) { | |
6a06907d | 910 | OptimizerLastEPCallbacks.push_back( |
94222f64 | 911 | [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) { |
94222f64 XL |
912 | HWAddressSanitizerOptions opts( |
913 | /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover, | |
914 | /*DisableOptimization=*/false); | |
915 | MPM.addPass(HWAddressSanitizerPass(opts)); | |
6a06907d XL |
916 | } |
917 | ); | |
74b04a01 XL |
918 | } |
919 | } | |
920 | ||
a2a8927a XL |
921 | if (LLVMPluginsLen) { |
922 | auto PluginsStr = StringRef(LLVMPlugins, LLVMPluginsLen); | |
923 | SmallVector<StringRef> Plugins; | |
924 | PluginsStr.split(Plugins, ',', -1, false); | |
925 | for (auto PluginPath: Plugins) { | |
926 | auto Plugin = PassPlugin::Load(PluginPath.str()); | |
927 | if (!Plugin) { | |
928 | LLVMRustSetLastError(("Failed to load pass plugin" + PluginPath.str()).c_str()); | |
487cf647 | 929 | return LLVMRustResult::Failure; |
a2a8927a XL |
930 | } |
931 | Plugin->registerPassBuilderCallbacks(PB); | |
932 | } | |
933 | } | |
934 | ||
17df50a5 | 935 | ModulePassManager MPM; |
6a06907d | 936 | bool NeedThinLTOBufferPasses = UseThinLTOBuffers; |
74b04a01 | 937 | if (!NoPrepopulatePasses) { |
49aad941 | 938 | // The pre-link pipelines don't support O0 and require using buildO0DefaultPipeline() instead. |
c295e0f8 XL |
939 | // At the same time, the LTO pipelines do support O0 and using them is required. |
940 | bool IsLTO = OptStage == LLVMRustOptStage::ThinLTO || OptStage == LLVMRustOptStage::FatLTO; | |
941 | if (OptLevel == OptimizationLevel::O0 && !IsLTO) { | |
74b04a01 | 942 | for (const auto &C : PipelineStartEPCallbacks) |
6a06907d XL |
943 | PB.registerPipelineStartEPCallback(C); |
944 | for (const auto &C : OptimizerLastEPCallbacks) | |
945 | PB.registerOptimizerLastEPCallback(C); | |
74b04a01 | 946 | |
6a06907d XL |
947 | // Pass false as we manually schedule ThinLTOBufferPasses below. |
948 | MPM = PB.buildO0DefaultPipeline(OptLevel, /* PreLinkLTO */ false); | |
74b04a01 XL |
949 | } else { |
950 | for (const auto &C : PipelineStartEPCallbacks) | |
951 | PB.registerPipelineStartEPCallback(C); | |
952 | if (OptStage != LLVMRustOptStage::PreLinkThinLTO) { | |
953 | for (const auto &C : OptimizerLastEPCallbacks) | |
954 | PB.registerOptimizerLastEPCallback(C); | |
955 | } | |
956 | ||
957 | switch (OptStage) { | |
958 | case LLVMRustOptStage::PreLinkNoLTO: | |
959 | MPM = PB.buildPerModuleDefaultPipeline(OptLevel, DebugPassManager); | |
960 | break; | |
961 | case LLVMRustOptStage::PreLinkThinLTO: | |
6a06907d XL |
962 | MPM = PB.buildThinLTOPreLinkDefaultPipeline(OptLevel); |
963 | // The ThinLTOPreLink pipeline already includes ThinLTOBuffer passes. However, callback | |
964 | // passes may still run afterwards. This means we need to run the buffer passes again. | |
965 | // FIXME: In LLVM 13, the ThinLTOPreLink pipeline also runs OptimizerLastEPCallbacks | |
966 | // before the RequiredLTOPreLinkPasses, in which case we can remove these hacks. | |
967 | if (OptimizerLastEPCallbacks.empty()) | |
968 | NeedThinLTOBufferPasses = false; | |
f035d41b XL |
969 | for (const auto &C : OptimizerLastEPCallbacks) |
970 | C(MPM, OptLevel); | |
74b04a01 XL |
971 | break; |
972 | case LLVMRustOptStage::PreLinkFatLTO: | |
6a06907d XL |
973 | MPM = PB.buildLTOPreLinkDefaultPipeline(OptLevel); |
974 | NeedThinLTOBufferPasses = false; | |
74b04a01 XL |
975 | break; |
976 | case LLVMRustOptStage::ThinLTO: | |
977 | // FIXME: Does it make sense to pass the ModuleSummaryIndex? | |
978 | // It only seems to be needed for C++ specific optimizations. | |
6a06907d | 979 | MPM = PB.buildThinLTODefaultPipeline(OptLevel, nullptr); |
74b04a01 XL |
980 | break; |
981 | case LLVMRustOptStage::FatLTO: | |
6a06907d | 982 | MPM = PB.buildLTODefaultPipeline(OptLevel, nullptr); |
74b04a01 XL |
983 | break; |
984 | } | |
985 | } | |
04454e1e FG |
986 | } else { |
987 | // We're not building any of the default pipelines but we still want to | |
988 | // add the verifier, instrumentation, etc passes if they were requested | |
989 | for (const auto &C : PipelineStartEPCallbacks) | |
990 | C(MPM, OptLevel); | |
991 | for (const auto &C : OptimizerLastEPCallbacks) | |
992 | C(MPM, OptLevel); | |
74b04a01 XL |
993 | } |
994 | ||
17df50a5 XL |
995 | if (ExtraPassesLen) { |
996 | if (auto Err = PB.parsePassPipeline(MPM, StringRef(ExtraPasses, ExtraPassesLen))) { | |
997 | std::string ErrMsg = toString(std::move(Err)); | |
998 | LLVMRustSetLastError(ErrMsg.c_str()); | |
999 | return LLVMRustResult::Failure; | |
1000 | } | |
1001 | } | |
1002 | ||
6a06907d | 1003 | if (NeedThinLTOBufferPasses) { |
74b04a01 XL |
1004 | MPM.addPass(CanonicalizeAliasesPass()); |
1005 | MPM.addPass(NameAnonGlobalPass()); | |
1006 | } | |
1007 | ||
1008 | // Upgrade all calls to old intrinsics first. | |
1009 | for (Module::iterator I = TheModule->begin(), E = TheModule->end(); I != E;) | |
1010 | UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove | |
1011 | ||
1012 | MPM.run(*TheModule, MAM); | |
17df50a5 | 1013 | return LLVMRustResult::Success; |
74b04a01 | 1014 | } |
041b39d2 XL |
1015 | |
1016 | // Callback to demangle function name | |
1017 | // Parameters: | |
1018 | // * name to be demangled | |
1019 | // * name len | |
1020 | // * output buffer | |
1021 | // * output buffer len | |
1022 | // Returns len of demangled string, or 0 if demangle failed. | |
1023 | typedef size_t (*DemangleFn)(const char*, size_t, char*, size_t); | |
1024 | ||
1025 | ||
1026 | namespace { | |
1027 | ||
1028 | class RustAssemblyAnnotationWriter : public AssemblyAnnotationWriter { | |
1029 | DemangleFn Demangle; | |
1030 | std::vector<char> Buf; | |
1031 | ||
1032 | public: | |
1033 | RustAssemblyAnnotationWriter(DemangleFn Demangle) : Demangle(Demangle) {} | |
1034 | ||
1035 | // Return empty string if demangle failed | |
1036 | // or if name does not need to be demangled | |
1037 | StringRef CallDemangle(StringRef name) { | |
1038 | if (!Demangle) { | |
1039 | return StringRef(); | |
1040 | } | |
1041 | ||
1042 | if (Buf.size() < name.size() * 2) { | |
1043 | // Semangled name usually shorter than mangled, | |
1044 | // but allocate twice as much memory just in case | |
1045 | Buf.resize(name.size() * 2); | |
1046 | } | |
1047 | ||
1048 | auto R = Demangle(name.data(), name.size(), Buf.data(), Buf.size()); | |
1049 | if (!R) { | |
1050 | // Demangle failed. | |
1051 | return StringRef(); | |
1052 | } | |
1053 | ||
1054 | auto Demangled = StringRef(Buf.data(), R); | |
1055 | if (Demangled == name) { | |
1056 | // Do not print anything if demangled name is equal to mangled. | |
1057 | return StringRef(); | |
1058 | } | |
1059 | ||
1060 | return Demangled; | |
1061 | } | |
1062 | ||
1063 | void emitFunctionAnnot(const Function *F, | |
1064 | formatted_raw_ostream &OS) override { | |
1065 | StringRef Demangled = CallDemangle(F->getName()); | |
1066 | if (Demangled.empty()) { | |
1067 | return; | |
1068 | } | |
1069 | ||
1070 | OS << "; " << Demangled << "\n"; | |
1071 | } | |
1072 | ||
1073 | void emitInstructionAnnot(const Instruction *I, | |
1074 | formatted_raw_ostream &OS) override { | |
1075 | const char *Name; | |
1076 | const Value *Value; | |
1077 | if (const CallInst *CI = dyn_cast<CallInst>(I)) { | |
1078 | Name = "call"; | |
f035d41b | 1079 | Value = CI->getCalledOperand(); |
041b39d2 XL |
1080 | } else if (const InvokeInst* II = dyn_cast<InvokeInst>(I)) { |
1081 | Name = "invoke"; | |
f035d41b | 1082 | Value = II->getCalledOperand(); |
041b39d2 XL |
1083 | } else { |
1084 | // Could demangle more operations, e. g. | |
1085 | // `store %place, @function`. | |
1086 | return; | |
1087 | } | |
1088 | ||
1089 | if (!Value->hasName()) { | |
1090 | return; | |
1091 | } | |
1092 | ||
1093 | StringRef Demangled = CallDemangle(Value->getName()); | |
1094 | if (Demangled.empty()) { | |
1095 | return; | |
1096 | } | |
1097 | ||
1098 | OS << "; " << Name << " " << Demangled << "\n"; | |
1099 | } | |
1100 | }; | |
1101 | ||
041b39d2 XL |
1102 | } // namespace |
1103 | ||
532ac7d7 | 1104 | extern "C" LLVMRustResult |
60c5eb7d | 1105 | LLVMRustPrintModule(LLVMModuleRef M, const char *Path, DemangleFn Demangle) { |
1a4d82fc | 1106 | std::string ErrorInfo; |
1a4d82fc | 1107 | std::error_code EC; |
17df50a5 | 1108 | raw_fd_ostream OS(Path, EC, sys::fs::OF_None); |
1a4d82fc JJ |
1109 | if (EC) |
1110 | ErrorInfo = EC.message(); | |
532ac7d7 XL |
1111 | if (ErrorInfo != "") { |
1112 | LLVMRustSetLastError(ErrorInfo.c_str()); | |
1113 | return LLVMRustResult::Failure; | |
1114 | } | |
1a4d82fc | 1115 | |
60c5eb7d | 1116 | RustAssemblyAnnotationWriter AAW(Demangle); |
1a4d82fc | 1117 | formatted_raw_ostream FOS(OS); |
60c5eb7d | 1118 | unwrap(M)->print(FOS, &AAW); |
532ac7d7 XL |
1119 | |
1120 | return LLVMRustResult::Success; | |
1a4d82fc JJ |
1121 | } |
1122 | ||
32a655c1 | 1123 | extern "C" void LLVMRustPrintPasses() { |
9ffffee4 FG |
1124 | PassBuilder PB; |
1125 | PB.printPassNames(outs()); | |
970d7e83 LB |
1126 | } |
1127 | ||
32a655c1 SL |
1128 | extern "C" void LLVMRustRunRestrictionPass(LLVMModuleRef M, char **Symbols, |
1129 | size_t Len) { | |
32a655c1 | 1130 | auto PreserveFunctions = [=](const GlobalValue &GV) { |
781aab86 FG |
1131 | // Preserve LLVM-injected, ASAN-related symbols. |
1132 | // See also https://github.com/rust-lang/rust/issues/113404. | |
1133 | if (GV.getName() == "___asan_globals_registered") { | |
1134 | return true; | |
1135 | } | |
1136 | ||
1137 | // Preserve symbols exported from Rust modules. | |
32a655c1 SL |
1138 | for (size_t I = 0; I < Len; I++) { |
1139 | if (GV.getName() == Symbols[I]) { | |
1140 | return true; | |
1141 | } | |
1142 | } | |
1143 | return false; | |
1144 | }; | |
5bcae85e | 1145 | |
2b03887a | 1146 | internalizeModule(*unwrap(M), PreserveFunctions); |
1a4d82fc JJ |
1147 | } |
1148 | ||
c1a9b12d SL |
1149 | extern "C" void |
1150 | LLVMRustSetDataLayoutFromTargetMachine(LLVMModuleRef Module, | |
1151 | LLVMTargetMachineRef TMR) { | |
32a655c1 SL |
1152 | TargetMachine *Target = unwrap(TMR); |
1153 | unwrap(Module)->setDataLayout(Target->createDataLayout()); | |
c1a9b12d SL |
1154 | } |
1155 | ||
60c5eb7d XL |
1156 | extern "C" void LLVMRustSetModulePICLevel(LLVMModuleRef M) { |
1157 | unwrap(M)->setPICLevel(PICLevel::Level::BigPIC); | |
1158 | } | |
1159 | ||
32a655c1 | 1160 | extern "C" void LLVMRustSetModulePIELevel(LLVMModuleRef M) { |
32a655c1 | 1161 | unwrap(M)->setPIELevel(PIELevel::Level::Large); |
5bcae85e | 1162 | } |
ea8adc8c | 1163 | |
6a06907d XL |
1164 | extern "C" void LLVMRustSetModuleCodeModel(LLVMModuleRef M, |
1165 | LLVMRustCodeModel Model) { | |
1166 | auto CM = fromRust(Model); | |
2b03887a | 1167 | if (!CM) |
6a06907d XL |
1168 | return; |
1169 | unwrap(M)->setCodeModel(*CM); | |
1170 | } | |
1171 | ||
ea8adc8c XL |
1172 | // Here you'll find an implementation of ThinLTO as used by the Rust compiler |
1173 | // right now. This ThinLTO support is only enabled on "recent ish" versions of | |
1174 | // LLVM, and otherwise it's just blanket rejected from other compilers. | |
1175 | // | |
1176 | // Most of this implementation is straight copied from LLVM. At the time of | |
1177 | // this writing it wasn't *quite* suitable to reuse more code from upstream | |
1178 | // for our purposes, but we should strive to upstream this support once it's | |
1179 | // ready to go! I figure we may want a bit of testing locally first before | |
1180 | // sending this upstream to LLVM. I hear though they're quite eager to receive | |
1181 | // feedback like this! | |
1182 | // | |
1183 | // If you're reading this code and wondering "what in the world" or you're | |
1184 | // working "good lord by LLVM upgrade is *still* failing due to these bindings" | |
1185 | // then fear not! (ok maybe fear a little). All code here is mostly based | |
1186 | // on `lib/LTO/ThinLTOCodeGenerator.cpp` in LLVM. | |
1187 | // | |
1188 | // You'll find that the general layout here roughly corresponds to the `run` | |
1189 | // method in that file as well as `ProcessThinLTOModule`. Functions are | |
1190 | // specifically commented below as well, but if you're updating this code | |
1191 | // or otherwise trying to understand it, the LLVM source will be useful in | |
1192 | // interpreting the mysteries within. | |
1193 | // | |
1194 | // Otherwise I'll apologize in advance, it probably requires a relatively | |
1195 | // significant investment on your part to "truly understand" what's going on | |
1196 | // here. Not saying I do myself, but it took me awhile staring at LLVM's source | |
1197 | // and various online resources about ThinLTO to make heads or tails of all | |
1198 | // this. | |
1199 | ||
ea8adc8c XL |
1200 | // This is a shared data structure which *must* be threadsafe to share |
1201 | // read-only amongst threads. This also corresponds basically to the arguments | |
1202 | // of the `ProcessThinLTOModule` function in the LLVM source. | |
1203 | struct LLVMRustThinLTOData { | |
1204 | // The combined index that is the global analysis over all modules we're | |
1205 | // performing ThinLTO for. This is mostly managed by LLVM. | |
1206 | ModuleSummaryIndex Index; | |
1207 | ||
1208 | // All modules we may look at, stored as in-memory serialized versions. This | |
1209 | // is later used when inlining to ensure we can extract any module to inline | |
1210 | // from. | |
1211 | StringMap<MemoryBufferRef> ModuleMap; | |
1212 | ||
1213 | // A set that we manage of everything we *don't* want internalized. Note that | |
1214 | // this includes all transitive references right now as well, but it may not | |
1215 | // always! | |
1216 | DenseSet<GlobalValue::GUID> GUIDPreservedSymbols; | |
1217 | ||
1218 | // Not 100% sure what these are, but they impact what's internalized and | |
1219 | // what's inlined across modules, I believe. | |
add651ee FG |
1220 | #if LLVM_VERSION_GE(18, 0) |
1221 | DenseMap<StringRef, FunctionImporter::ImportMapTy> ImportLists; | |
1222 | DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists; | |
1223 | DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries; | |
1224 | #else | |
ea8adc8c XL |
1225 | StringMap<FunctionImporter::ImportMapTy> ImportLists; |
1226 | StringMap<FunctionImporter::ExportSetTy> ExportLists; | |
1227 | StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries; | |
add651ee | 1228 | #endif |
29967ef6 | 1229 | StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR; |
2c00a5a8 | 1230 | |
9fa01778 | 1231 | LLVMRustThinLTOData() : Index(/* HaveGVs = */ false) {} |
ea8adc8c XL |
1232 | }; |
1233 | ||
1234 | // Just an argument to the `LLVMRustCreateThinLTOData` function below. | |
1235 | struct LLVMRustThinLTOModule { | |
1236 | const char *identifier; | |
1237 | const char *data; | |
1238 | size_t len; | |
1239 | }; | |
1240 | ||
1241 | // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`, not sure what it | |
1242 | // does. | |
1243 | static const GlobalValueSummary * | |
1244 | getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) { | |
1245 | auto StrongDefForLinker = llvm::find_if( | |
1246 | GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) { | |
1247 | auto Linkage = Summary->linkage(); | |
1248 | return !GlobalValue::isAvailableExternallyLinkage(Linkage) && | |
1249 | !GlobalValue::isWeakForLinker(Linkage); | |
1250 | }); | |
1251 | if (StrongDefForLinker != GVSummaryList.end()) | |
1252 | return StrongDefForLinker->get(); | |
1253 | ||
1254 | auto FirstDefForLinker = llvm::find_if( | |
1255 | GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) { | |
1256 | auto Linkage = Summary->linkage(); | |
1257 | return !GlobalValue::isAvailableExternallyLinkage(Linkage); | |
1258 | }); | |
1259 | if (FirstDefForLinker == GVSummaryList.end()) | |
1260 | return nullptr; | |
1261 | return FirstDefForLinker->get(); | |
1262 | } | |
1263 | ||
ea8adc8c XL |
1264 | // The main entry point for creating the global ThinLTO analysis. The structure |
1265 | // here is basically the same as before threads are spawned in the `run` | |
1266 | // function of `lib/LTO/ThinLTOCodeGenerator.cpp`. | |
1267 | extern "C" LLVMRustThinLTOData* | |
1268 | LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules, | |
1269 | int num_modules, | |
1270 | const char **preserved_symbols, | |
1271 | int num_symbols) { | |
dfeec247 | 1272 | auto Ret = std::make_unique<LLVMRustThinLTOData>(); |
ea8adc8c XL |
1273 | |
1274 | // Load each module's summary and merge it into one combined index | |
1275 | for (int i = 0; i < num_modules; i++) { | |
1276 | auto module = &modules[i]; | |
1277 | StringRef buffer(module->data, module->len); | |
1278 | MemoryBufferRef mem_buffer(buffer, module->identifier); | |
1279 | ||
1280 | Ret->ModuleMap[module->identifier] = mem_buffer; | |
1281 | ||
781aab86 FG |
1282 | #if LLVM_VERSION_GE(18, 0) |
1283 | if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index)) { | |
1284 | #else | |
abe05a73 | 1285 | if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index, i)) { |
781aab86 | 1286 | #endif |
abe05a73 XL |
1287 | LLVMRustSetLastError(toString(std::move(Err)).c_str()); |
1288 | return nullptr; | |
1289 | } | |
ea8adc8c XL |
1290 | } |
1291 | ||
1292 | // Collect for each module the list of function it defines (GUID -> Summary) | |
1293 | Ret->Index.collectDefinedGVSummariesPerModule(Ret->ModuleToDefinedGVSummaries); | |
1294 | ||
1295 | // Convert the preserved symbols set from string to GUID, this is then needed | |
ff7c6d11 | 1296 | // for internalization. |
ea8adc8c | 1297 | for (int i = 0; i < num_symbols; i++) { |
ff7c6d11 XL |
1298 | auto GUID = GlobalValue::getGUID(preserved_symbols[i]); |
1299 | Ret->GUIDPreservedSymbols.insert(GUID); | |
ea8adc8c XL |
1300 | } |
1301 | ||
1302 | // Collect the import/export lists for all modules from the call-graph in the | |
1303 | // combined index | |
1304 | // | |
1305 | // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` | |
2c00a5a8 XL |
1306 | auto deadIsPrevailing = [&](GlobalValue::GUID G) { |
1307 | return PrevailingType::Unknown; | |
1308 | }; | |
48663c56 XL |
1309 | // We don't have a complete picture in our use of ThinLTO, just our immediate |
1310 | // crate, so we need `ImportEnabled = false` to limit internalization. | |
1311 | // Otherwise, we sometimes lose `static` values -- see #60184. | |
9fa01778 | 1312 | computeDeadSymbolsWithConstProp(Ret->Index, Ret->GUIDPreservedSymbols, |
48663c56 | 1313 | deadIsPrevailing, /* ImportEnabled = */ false); |
ea8adc8c XL |
1314 | // Resolve LinkOnce/Weak symbols, this has to be computed early be cause it |
1315 | // impacts the caching. | |
1316 | // | |
ff7c6d11 XL |
1317 | // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` with some of this |
1318 | // being lifted from `lib/LTO/LTO.cpp` as well | |
ea8adc8c XL |
1319 | DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy; |
1320 | for (auto &I : Ret->Index) { | |
abe05a73 XL |
1321 | if (I.second.SummaryList.size() > 1) |
1322 | PrevailingCopy[I.first] = getFirstDefinitionForLinker(I.second.SummaryList); | |
ea8adc8c XL |
1323 | } |
1324 | auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) { | |
1325 | const auto &Prevailing = PrevailingCopy.find(GUID); | |
1326 | if (Prevailing == PrevailingCopy.end()) | |
1327 | return true; | |
1328 | return Prevailing->second == S; | |
1329 | }; | |
353b0b11 FG |
1330 | ComputeCrossModuleImport( |
1331 | Ret->Index, | |
1332 | Ret->ModuleToDefinedGVSummaries, | |
1333 | #if LLVM_VERSION_GE(17, 0) | |
1334 | isPrevailing, | |
1335 | #endif | |
1336 | Ret->ImportLists, | |
1337 | Ret->ExportLists | |
1338 | ); | |
1339 | ||
ea8adc8c XL |
1340 | auto recordNewLinkage = [&](StringRef ModuleIdentifier, |
1341 | GlobalValue::GUID GUID, | |
1342 | GlobalValue::LinkageTypes NewLinkage) { | |
29967ef6 | 1343 | Ret->ResolvedODR[ModuleIdentifier][GUID] = NewLinkage; |
ea8adc8c | 1344 | }; |
fc512014 | 1345 | |
cdc7bbd5 XL |
1346 | // Uses FromPrevailing visibility scheme which works for many binary |
1347 | // formats. We probably could and should use ELF visibility scheme for many of | |
1348 | // our targets, however. | |
1349 | lto::Config conf; | |
1350 | thinLTOResolvePrevailingInIndex(conf, Ret->Index, isPrevailing, recordNewLinkage, | |
1351 | Ret->GUIDPreservedSymbols); | |
f2b60f7d | 1352 | |
ff7c6d11 XL |
1353 | // Here we calculate an `ExportedGUIDs` set for use in the `isExported` |
1354 | // callback below. This callback below will dictate the linkage for all | |
1355 | // summaries in the index, and we basically just only want to ensure that dead | |
1356 | // symbols are internalized. Otherwise everything that's already external | |
1357 | // linkage will stay as external, and internal will stay as internal. | |
1358 | std::set<GlobalValue::GUID> ExportedGUIDs; | |
1359 | for (auto &List : Ret->Index) { | |
ff7c6d11 | 1360 | for (auto &GVS: List.second.SummaryList) { |
ff7c6d11 XL |
1361 | if (GlobalValue::isLocalLinkage(GVS->linkage())) |
1362 | continue; | |
1363 | auto GUID = GVS->getOriginalName(); | |
ff7c6d11 | 1364 | if (GVS->flags().Live) |
ff7c6d11 XL |
1365 | ExportedGUIDs.insert(GUID); |
1366 | } | |
1367 | } | |
dfeec247 XL |
1368 | auto isExported = [&](StringRef ModuleIdentifier, ValueInfo VI) { |
1369 | const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier); | |
1370 | return (ExportList != Ret->ExportLists.end() && | |
1371 | ExportList->second.count(VI)) || | |
1372 | ExportedGUIDs.count(VI.getGUID()); | |
1373 | }; | |
1374 | thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported, isPrevailing); | |
ea8adc8c XL |
1375 | |
1376 | return Ret.release(); | |
1377 | } | |
1378 | ||
1379 | extern "C" void | |
1380 | LLVMRustFreeThinLTOData(LLVMRustThinLTOData *Data) { | |
1381 | delete Data; | |
1382 | } | |
1383 | ||
1384 | // Below are the various passes that happen *per module* when doing ThinLTO. | |
1385 | // | |
1386 | // In other words, these are the functions that are all run concurrently | |
1387 | // with one another, one per module. The passes here correspond to the analysis | |
1388 | // passes in `lib/LTO/ThinLTOCodeGenerator.cpp`, currently found in the | |
1389 | // `ProcessThinLTOModule` function. Here they're split up into separate steps | |
1390 | // so rustc can save off the intermediate bytecode between each step. | |
1391 | ||
f035d41b XL |
1392 | static bool |
1393 | clearDSOLocalOnDeclarations(Module &Mod, TargetMachine &TM) { | |
1394 | // When linking an ELF shared object, dso_local should be dropped. We | |
1395 | // conservatively do this for -fpic. | |
1396 | bool ClearDSOLocalOnDeclarations = | |
1397 | TM.getTargetTriple().isOSBinFormatELF() && | |
1398 | TM.getRelocationModel() != Reloc::Static && | |
1399 | Mod.getPIELevel() == PIELevel::Default; | |
1400 | return ClearDSOLocalOnDeclarations; | |
1401 | } | |
f035d41b | 1402 | |
ea8adc8c | 1403 | extern "C" bool |
f035d41b XL |
1404 | LLVMRustPrepareThinLTORename(const LLVMRustThinLTOData *Data, LLVMModuleRef M, |
1405 | LLVMTargetMachineRef TM) { | |
ea8adc8c | 1406 | Module &Mod = *unwrap(M); |
f035d41b XL |
1407 | TargetMachine &Target = *unwrap(TM); |
1408 | ||
f035d41b XL |
1409 | bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target); |
1410 | bool error = renameModuleForThinLTO(Mod, Data->Index, ClearDSOLocal); | |
f035d41b XL |
1411 | |
1412 | if (error) { | |
ea8adc8c XL |
1413 | LLVMRustSetLastError("renameModuleForThinLTO failed"); |
1414 | return false; | |
1415 | } | |
1416 | return true; | |
1417 | } | |
1418 | ||
1419 | extern "C" bool | |
1420 | LLVMRustPrepareThinLTOResolveWeak(const LLVMRustThinLTOData *Data, LLVMModuleRef M) { | |
1421 | Module &Mod = *unwrap(M); | |
1422 | const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier()); | |
c295e0f8 | 1423 | thinLTOFinalizeInModule(Mod, DefinedGlobals, /*PropagateAttrs=*/true); |
ea8adc8c XL |
1424 | return true; |
1425 | } | |
1426 | ||
1427 | extern "C" bool | |
1428 | LLVMRustPrepareThinLTOInternalize(const LLVMRustThinLTOData *Data, LLVMModuleRef M) { | |
1429 | Module &Mod = *unwrap(M); | |
1430 | const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier()); | |
1431 | thinLTOInternalizeModule(Mod, DefinedGlobals); | |
1432 | return true; | |
1433 | } | |
1434 | ||
1435 | extern "C" bool | |
f035d41b XL |
1436 | LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, LLVMModuleRef M, |
1437 | LLVMTargetMachineRef TM) { | |
ea8adc8c | 1438 | Module &Mod = *unwrap(M); |
f035d41b | 1439 | TargetMachine &Target = *unwrap(TM); |
8faf50e0 | 1440 | |
ea8adc8c XL |
1441 | const auto &ImportList = Data->ImportLists.lookup(Mod.getModuleIdentifier()); |
1442 | auto Loader = [&](StringRef Identifier) { | |
1443 | const auto &Memory = Data->ModuleMap.lookup(Identifier); | |
1444 | auto &Context = Mod.getContext(); | |
8faf50e0 XL |
1445 | auto MOrErr = getLazyBitcodeModule(Memory, Context, true, true); |
1446 | ||
1447 | if (!MOrErr) | |
b7449926 | 1448 | return MOrErr; |
8faf50e0 XL |
1449 | |
1450 | // The rest of this closure is a workaround for | |
1451 | // https://bugs.llvm.org/show_bug.cgi?id=38184 where during ThinLTO imports | |
1452 | // we accidentally import wasm custom sections into different modules, | |
1453 | // duplicating them by in the final output artifact. | |
1454 | // | |
1455 | // The issue is worked around here by manually removing the | |
1456 | // `wasm.custom_sections` named metadata node from any imported module. This | |
1457 | // we know isn't used by any optimization pass so there's no need for it to | |
1458 | // be imported. | |
1459 | // | |
1460 | // Note that the metadata is currently lazily loaded, so we materialize it | |
1461 | // here before looking up if there's metadata inside. The `FunctionImporter` | |
1462 | // will immediately materialize metadata anyway after an import, so this | |
1463 | // shouldn't be a perf hit. | |
1464 | if (Error Err = (*MOrErr)->materializeMetadata()) { | |
1465 | Expected<std::unique_ptr<Module>> Ret(std::move(Err)); | |
b7449926 | 1466 | return Ret; |
8faf50e0 XL |
1467 | } |
1468 | ||
1469 | auto *WasmCustomSections = (*MOrErr)->getNamedMetadata("wasm.custom_sections"); | |
1470 | if (WasmCustomSections) | |
1471 | WasmCustomSections->eraseFromParent(); | |
1472 | ||
add651ee FG |
1473 | // `llvm.ident` named metadata also gets duplicated. |
1474 | auto *llvmIdent = (*MOrErr)->getNamedMetadata("llvm.ident"); | |
1475 | if (llvmIdent) | |
1476 | llvmIdent->eraseFromParent(); | |
1477 | ||
b7449926 | 1478 | return MOrErr; |
ea8adc8c | 1479 | }; |
f035d41b XL |
1480 | bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target); |
1481 | FunctionImporter Importer(Data->Index, Loader, ClearDSOLocal); | |
ea8adc8c XL |
1482 | Expected<bool> Result = Importer.importFunctions(Mod, ImportList); |
1483 | if (!Result) { | |
1484 | LLVMRustSetLastError(toString(Result.takeError()).c_str()); | |
1485 | return false; | |
1486 | } | |
1487 | return true; | |
1488 | } | |
1489 | ||
1490 | // This struct and various functions are sort of a hack right now, but the | |
1491 | // problem is that we've got in-memory LLVM modules after we generate and | |
1492 | // optimize all codegen-units for one compilation in rustc. To be compatible | |
1493 | // with the LTO support above we need to serialize the modules plus their | |
1494 | // ThinLTO summary into memory. | |
1495 | // | |
1496 | // This structure is basically an owned version of a serialize module, with | |
1497 | // a ThinLTO summary attached. | |
1498 | struct LLVMRustThinLTOBuffer { | |
1499 | std::string data; | |
1500 | }; | |
1501 | ||
1502 | extern "C" LLVMRustThinLTOBuffer* | |
064997fb | 1503 | LLVMRustThinLTOBufferCreate(LLVMModuleRef M, bool is_thin) { |
dfeec247 | 1504 | auto Ret = std::make_unique<LLVMRustThinLTOBuffer>(); |
ea8adc8c XL |
1505 | { |
1506 | raw_string_ostream OS(Ret->data); | |
1507 | { | |
064997fb | 1508 | if (is_thin) { |
f2b60f7d FG |
1509 | PassBuilder PB; |
1510 | LoopAnalysisManager LAM; | |
1511 | FunctionAnalysisManager FAM; | |
1512 | CGSCCAnalysisManager CGAM; | |
1513 | ModuleAnalysisManager MAM; | |
1514 | PB.registerModuleAnalyses(MAM); | |
1515 | PB.registerCGSCCAnalyses(CGAM); | |
1516 | PB.registerFunctionAnalyses(FAM); | |
1517 | PB.registerLoopAnalyses(LAM); | |
1518 | PB.crossRegisterProxies(LAM, FAM, CGAM, MAM); | |
1519 | ModulePassManager MPM; | |
1520 | MPM.addPass(ThinLTOBitcodeWriterPass(OS, nullptr)); | |
1521 | MPM.run(*unwrap(M), MAM); | |
064997fb | 1522 | } else { |
2b03887a | 1523 | WriteBitcodeToFile(*unwrap(M), OS); |
064997fb | 1524 | } |
ea8adc8c XL |
1525 | } |
1526 | } | |
1527 | return Ret.release(); | |
1528 | } | |
1529 | ||
1530 | extern "C" void | |
1531 | LLVMRustThinLTOBufferFree(LLVMRustThinLTOBuffer *Buffer) { | |
1532 | delete Buffer; | |
1533 | } | |
1534 | ||
1535 | extern "C" const void* | |
1536 | LLVMRustThinLTOBufferPtr(const LLVMRustThinLTOBuffer *Buffer) { | |
1537 | return Buffer->data.data(); | |
1538 | } | |
1539 | ||
1540 | extern "C" size_t | |
1541 | LLVMRustThinLTOBufferLen(const LLVMRustThinLTOBuffer *Buffer) { | |
1542 | return Buffer->data.length(); | |
1543 | } | |
1544 | ||
1545 | // This is what we used to parse upstream bitcode for actual ThinLTO | |
9c376795 | 1546 | // processing. We'll call this once per module optimized through ThinLTO, and |
ea8adc8c XL |
1547 | // it'll be called concurrently on many threads. |
1548 | extern "C" LLVMModuleRef | |
9fa01778 XL |
1549 | LLVMRustParseBitcodeForLTO(LLVMContextRef Context, |
1550 | const char *data, | |
1551 | size_t len, | |
1552 | const char *identifier) { | |
ea8adc8c XL |
1553 | StringRef Data(data, len); |
1554 | MemoryBufferRef Buffer(Data, identifier); | |
1555 | unwrap(Context)->enableDebugTypeODRUniquing(); | |
1556 | Expected<std::unique_ptr<Module>> SrcOrError = | |
1557 | parseBitcodeFile(Buffer, *unwrap(Context)); | |
1558 | if (!SrcOrError) { | |
1559 | LLVMRustSetLastError(toString(SrcOrError.takeError()).c_str()); | |
1560 | return nullptr; | |
1561 | } | |
1562 | return wrap(std::move(*SrcOrError).release()); | |
1563 | } | |
1564 | ||
f9f354fc XL |
1565 | // Find the bitcode section in the object file data and return it as a slice. |
1566 | // Fail if the bitcode section is present but empty. | |
1567 | // | |
1568 | // On success, the return value is the pointer to the start of the slice and | |
1569 | // `out_len` is filled with the (non-zero) length. On failure, the return value | |
1570 | // is `nullptr` and `out_len` is set to zero. | |
1571 | extern "C" const char* | |
1572 | LLVMRustGetBitcodeSliceFromObjectData(const char *data, | |
1573 | size_t len, | |
1574 | size_t *out_len) { | |
1575 | *out_len = 0; | |
1576 | ||
1577 | StringRef Data(data, len); | |
1578 | MemoryBufferRef Buffer(Data, ""); // The id is unused. | |
1579 | ||
1580 | Expected<MemoryBufferRef> BitcodeOrError = | |
1581 | object::IRObjectFile::findBitcodeInMemBuffer(Buffer); | |
1582 | if (!BitcodeOrError) { | |
1583 | LLVMRustSetLastError(toString(BitcodeOrError.takeError()).c_str()); | |
1584 | return nullptr; | |
1585 | } | |
1586 | ||
1587 | *out_len = BitcodeOrError->getBufferSize(); | |
1588 | return BitcodeOrError->getBufferStart(); | |
1589 | } | |
1590 | ||
781aab86 FG |
1591 | // Find a section of an object file by name. Fail if the section is missing or |
1592 | // empty. | |
1593 | extern "C" const char *LLVMRustGetSliceFromObjectDataByName(const char *data, | |
1594 | size_t len, | |
1595 | const char *name, | |
1596 | size_t *out_len) { | |
1597 | *out_len = 0; | |
1598 | StringRef Data(data, len); | |
1599 | MemoryBufferRef Buffer(Data, ""); // The id is unused. | |
1600 | file_magic Type = identify_magic(Buffer.getBuffer()); | |
1601 | Expected<std::unique_ptr<object::ObjectFile>> ObjFileOrError = | |
1602 | object::ObjectFile::createObjectFile(Buffer, Type); | |
1603 | if (!ObjFileOrError) { | |
1604 | LLVMRustSetLastError(toString(ObjFileOrError.takeError()).c_str()); | |
1605 | return nullptr; | |
1606 | } | |
1607 | for (const object::SectionRef &Sec : (*ObjFileOrError)->sections()) { | |
1608 | Expected<StringRef> Name = Sec.getName(); | |
1609 | if (Name && *Name == name) { | |
1610 | Expected<StringRef> SectionOrError = Sec.getContents(); | |
1611 | if (!SectionOrError) { | |
1612 | LLVMRustSetLastError(toString(SectionOrError.takeError()).c_str()); | |
1613 | return nullptr; | |
1614 | } | |
1615 | *out_len = SectionOrError->size(); | |
1616 | return SectionOrError->data(); | |
1617 | } | |
1618 | } | |
1619 | LLVMRustSetLastError("could not find requested section"); | |
1620 | return nullptr; | |
1621 | } | |
1622 | ||
29967ef6 XL |
1623 | // Computes the LTO cache key for the provided 'ModId' in the given 'Data', |
1624 | // storing the result in 'KeyOut'. | |
1625 | // Currently, this cache key is a SHA-1 hash of anything that could affect | |
1626 | // the result of optimizing this module (e.g. module imports, exports, liveness | |
1627 | // of access globals, etc). | |
1628 | // The precise details are determined by LLVM in `computeLTOCacheKey`, which is | |
1629 | // used during the normal linker-plugin incremental thin-LTO process. | |
1630 | extern "C" void | |
1631 | LLVMRustComputeLTOCacheKey(RustStringRef KeyOut, const char *ModId, LLVMRustThinLTOData *Data) { | |
1632 | SmallString<40> Key; | |
1633 | llvm::lto::Config conf; | |
1634 | const auto &ImportList = Data->ImportLists.lookup(ModId); | |
1635 | const auto &ExportList = Data->ExportLists.lookup(ModId); | |
1636 | const auto &ResolvedODR = Data->ResolvedODR.lookup(ModId); | |
1637 | const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(ModId); | |
1638 | std::set<GlobalValue::GUID> CfiFunctionDefs; | |
1639 | std::set<GlobalValue::GUID> CfiFunctionDecls; | |
1640 | ||
1641 | // Based on the 'InProcessThinBackend' constructor in LLVM | |
1642 | for (auto &Name : Data->Index.cfiFunctionDefs()) | |
1643 | CfiFunctionDefs.insert( | |
1644 | GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name))); | |
1645 | for (auto &Name : Data->Index.cfiFunctionDecls()) | |
1646 | CfiFunctionDecls.insert( | |
1647 | GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name))); | |
1648 | ||
1649 | llvm::computeLTOCacheKey(Key, conf, Data->Index, ModId, | |
1650 | ImportList, ExportList, ResolvedODR, DefinedGlobals, CfiFunctionDefs, CfiFunctionDecls | |
1651 | ); | |
1652 | ||
1653 | LLVMRustStringWriteImpl(KeyOut, Key.c_str(), Key.size()); | |
1654 | } |