6 #include "LLVMWrapper.h"
8 #include "llvm/Analysis/AliasAnalysis.h"
9 #include "llvm/Analysis/TargetLibraryInfo.h"
10 #include "llvm/Analysis/TargetTransformInfo.h"
11 #include "llvm/CodeGen/TargetSubtargetInfo.h"
12 #include "llvm/InitializePasses.h"
13 #include "llvm/IR/AutoUpgrade.h"
14 #include "llvm/IR/AssemblyAnnotationWriter.h"
15 #include "llvm/IR/IntrinsicInst.h"
16 #include "llvm/IR/Verifier.h"
17 #include "llvm/Object/ObjectFile.h"
18 #include "llvm/Object/IRObjectFile.h"
19 #include "llvm/Passes/PassBuilder.h"
20 #include "llvm/Passes/PassPlugin.h"
21 #include "llvm/Passes/StandardInstrumentations.h"
22 #include "llvm/Support/CBindingWrapping.h"
23 #include "llvm/Support/FileSystem.h"
24 #include "llvm/Support/Host.h"
25 #if LLVM_VERSION_LT(14, 0)
26 #include "llvm/Support/TargetRegistry.h"
28 #include "llvm/MC/TargetRegistry.h"
30 #include "llvm/Target/TargetMachine.h"
31 #include "llvm/Transforms/IPO/PassManagerBuilder.h"
32 #include "llvm/Transforms/IPO/AlwaysInliner.h"
33 #include "llvm/Transforms/IPO/FunctionImport.h"
34 #include "llvm/Transforms/IPO/Internalize.h"
35 #include "llvm/Transforms/IPO/ThinLTOBitcodeWriter.h"
36 #include "llvm/Transforms/Utils/AddDiscriminators.h"
37 #include "llvm/Transforms/Utils/FunctionImportUtils.h"
38 #include "llvm/LTO/LTO.h"
39 #include "llvm/Bitcode/BitcodeWriter.h"
40 #include "llvm-c/Transforms/PassManagerBuilder.h"
42 #include "llvm/Transforms/Instrumentation.h"
43 #include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
44 #include "llvm/Support/TimeProfiler.h"
45 #include "llvm/Transforms/Instrumentation/GCOVProfiler.h"
46 #include "llvm/Transforms/Instrumentation/InstrProfiling.h"
47 #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h"
48 #include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
49 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
50 #include "llvm/Transforms/Utils/CanonicalizeAliases.h"
51 #include "llvm/Transforms/Utils/NameAnonGlobals.h"
52 #include "llvm/Transforms/Utils.h"
56 typedef struct LLVMOpaquePass
*LLVMPassRef
;
57 typedef struct LLVMOpaqueTargetMachine
*LLVMTargetMachineRef
;
59 DEFINE_STDCXX_CONVERSION_FUNCTIONS(Pass
, LLVMPassRef
)
60 DEFINE_STDCXX_CONVERSION_FUNCTIONS(TargetMachine
, LLVMTargetMachineRef
)
62 extern "C" void LLVMInitializePasses() {
63 PassRegistry
&Registry
= *PassRegistry::getPassRegistry();
64 initializeCore(Registry
);
65 initializeCodeGen(Registry
);
66 initializeScalarOpts(Registry
);
67 initializeVectorization(Registry
);
68 initializeIPO(Registry
);
69 initializeAnalysis(Registry
);
70 initializeTransformUtils(Registry
);
71 initializeInstCombine(Registry
);
72 #if LLVM_VERSION_LT(16, 0)
73 initializeInstrumentation(Registry
);
75 initializeTarget(Registry
);
78 extern "C" void LLVMTimeTraceProfilerInitialize() {
79 timeTraceProfilerInitialize(
80 /* TimeTraceGranularity */ 0,
81 /* ProcName */ "rustc");
84 extern "C" void LLVMTimeTraceProfilerFinishThread() {
85 timeTraceProfilerFinishThread();
88 extern "C" void LLVMTimeTraceProfilerFinish(const char* FileName
) {
89 StringRef
FN(FileName
);
91 raw_fd_ostream
OS(FN
, EC
, sys::fs::CD_CreateAlways
);
93 timeTraceProfilerWrite(OS
);
94 timeTraceProfilerCleanup();
97 #ifdef LLVM_COMPONENT_X86
98 #define SUBTARGET_X86 SUBTARGET(X86)
100 #define SUBTARGET_X86
103 #ifdef LLVM_COMPONENT_ARM
104 #define SUBTARGET_ARM SUBTARGET(ARM)
106 #define SUBTARGET_ARM
109 #ifdef LLVM_COMPONENT_AARCH64
110 #define SUBTARGET_AARCH64 SUBTARGET(AArch64)
112 #define SUBTARGET_AARCH64
115 #ifdef LLVM_COMPONENT_AVR
116 #define SUBTARGET_AVR SUBTARGET(AVR)
118 #define SUBTARGET_AVR
121 #ifdef LLVM_COMPONENT_M68k
122 #define SUBTARGET_M68K SUBTARGET(M68k)
124 #define SUBTARGET_M68K
127 #ifdef LLVM_COMPONENT_MIPS
128 #define SUBTARGET_MIPS SUBTARGET(Mips)
130 #define SUBTARGET_MIPS
133 #ifdef LLVM_COMPONENT_POWERPC
134 #define SUBTARGET_PPC SUBTARGET(PPC)
136 #define SUBTARGET_PPC
139 #ifdef LLVM_COMPONENT_SYSTEMZ
140 #define SUBTARGET_SYSTEMZ SUBTARGET(SystemZ)
142 #define SUBTARGET_SYSTEMZ
145 #ifdef LLVM_COMPONENT_MSP430
146 #define SUBTARGET_MSP430 SUBTARGET(MSP430)
148 #define SUBTARGET_MSP430
151 #ifdef LLVM_COMPONENT_RISCV
152 #define SUBTARGET_RISCV SUBTARGET(RISCV)
154 #define SUBTARGET_RISCV
157 #ifdef LLVM_COMPONENT_SPARC
158 #define SUBTARGET_SPARC SUBTARGET(Sparc)
160 #define SUBTARGET_SPARC
163 #ifdef LLVM_COMPONENT_HEXAGON
164 #define SUBTARGET_HEXAGON SUBTARGET(Hexagon)
166 #define SUBTARGET_HEXAGON
169 #define GEN_SUBTARGETS \
183 #define SUBTARGET(x) \
185 extern const SubtargetFeatureKV x##FeatureKV[]; \
186 extern const SubtargetFeatureKV x##SubTypeKV[]; \
192 extern "C" bool LLVMRustHasFeature(LLVMTargetMachineRef TM
,
193 const char *Feature
) {
194 TargetMachine
*Target
= unwrap(TM
);
195 const MCSubtargetInfo
*MCInfo
= Target
->getMCSubtargetInfo();
196 return MCInfo
->checkFeatures(std::string("+") + Feature
);
199 enum class LLVMRustCodeModel
{
208 #if LLVM_VERSION_LT(16, 0)
209 static Optional
<CodeModel::Model
>
211 static std::optional
<CodeModel::Model
>
213 fromRust(LLVMRustCodeModel Model
) {
215 case LLVMRustCodeModel::Tiny
:
216 return CodeModel::Tiny
;
217 case LLVMRustCodeModel::Small
:
218 return CodeModel::Small
;
219 case LLVMRustCodeModel::Kernel
:
220 return CodeModel::Kernel
;
221 case LLVMRustCodeModel::Medium
:
222 return CodeModel::Medium
;
223 case LLVMRustCodeModel::Large
:
224 return CodeModel::Large
;
225 case LLVMRustCodeModel::None
:
228 report_fatal_error("Bad CodeModel.");
232 enum class LLVMRustCodeGenOptLevel
{
239 static CodeGenOpt::Level
fromRust(LLVMRustCodeGenOptLevel Level
) {
241 case LLVMRustCodeGenOptLevel::None
:
242 return CodeGenOpt::None
;
243 case LLVMRustCodeGenOptLevel::Less
:
244 return CodeGenOpt::Less
;
245 case LLVMRustCodeGenOptLevel::Default
:
246 return CodeGenOpt::Default
;
247 case LLVMRustCodeGenOptLevel::Aggressive
:
248 return CodeGenOpt::Aggressive
;
250 report_fatal_error("Bad CodeGenOptLevel.");
254 enum class LLVMRustPassBuilderOptLevel
{
263 #if LLVM_VERSION_LT(14,0)
264 using OptimizationLevel
= PassBuilder::OptimizationLevel
;
267 static OptimizationLevel
fromRust(LLVMRustPassBuilderOptLevel Level
) {
269 case LLVMRustPassBuilderOptLevel::O0
:
270 return OptimizationLevel::O0
;
271 case LLVMRustPassBuilderOptLevel::O1
:
272 return OptimizationLevel::O1
;
273 case LLVMRustPassBuilderOptLevel::O2
:
274 return OptimizationLevel::O2
;
275 case LLVMRustPassBuilderOptLevel::O3
:
276 return OptimizationLevel::O3
;
277 case LLVMRustPassBuilderOptLevel::Os
:
278 return OptimizationLevel::Os
;
279 case LLVMRustPassBuilderOptLevel::Oz
:
280 return OptimizationLevel::Oz
;
282 report_fatal_error("Bad PassBuilderOptLevel.");
286 enum class LLVMRustRelocModel
{
295 static Reloc::Model
fromRust(LLVMRustRelocModel RustReloc
) {
297 case LLVMRustRelocModel::Static
:
298 return Reloc::Static
;
299 case LLVMRustRelocModel::PIC
:
301 case LLVMRustRelocModel::DynamicNoPic
:
302 return Reloc::DynamicNoPIC
;
303 case LLVMRustRelocModel::ROPI
:
305 case LLVMRustRelocModel::RWPI
:
307 case LLVMRustRelocModel::ROPIRWPI
:
308 return Reloc::ROPI_RWPI
;
310 report_fatal_error("Bad RelocModel.");
314 /// getLongestEntryLength - Return the length of the longest entry in the table.
315 template<typename KV
>
316 static size_t getLongestEntryLength(ArrayRef
<KV
> Table
) {
318 for (auto &I
: Table
)
319 MaxLen
= std::max(MaxLen
, std::strlen(I
.Key
));
323 extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef TM
) {
324 const TargetMachine
*Target
= unwrap(TM
);
325 const MCSubtargetInfo
*MCInfo
= Target
->getMCSubtargetInfo();
326 const Triple::ArchType HostArch
= Triple(sys::getProcessTriple()).getArch();
327 const Triple::ArchType TargetArch
= Target
->getTargetTriple().getArch();
328 const ArrayRef
<SubtargetSubTypeKV
> CPUTable
= MCInfo
->getCPUTable();
329 unsigned MaxCPULen
= getLongestEntryLength(CPUTable
);
331 printf("Available CPUs for this target:\n");
332 if (HostArch
== TargetArch
) {
333 const StringRef HostCPU
= sys::getHostCPUName();
334 printf(" %-*s - Select the CPU of the current host (currently %.*s).\n",
335 MaxCPULen
, "native", (int)HostCPU
.size(), HostCPU
.data());
337 for (auto &CPU
: CPUTable
)
338 printf(" %-*s\n", MaxCPULen
, CPU
.Key
);
342 extern "C" size_t LLVMRustGetTargetFeaturesCount(LLVMTargetMachineRef TM
) {
343 const TargetMachine
*Target
= unwrap(TM
);
344 const MCSubtargetInfo
*MCInfo
= Target
->getMCSubtargetInfo();
345 const ArrayRef
<SubtargetFeatureKV
> FeatTable
= MCInfo
->getFeatureTable();
346 return FeatTable
.size();
349 extern "C" void LLVMRustGetTargetFeature(LLVMTargetMachineRef TM
, size_t Index
,
350 const char** Feature
, const char** Desc
) {
351 const TargetMachine
*Target
= unwrap(TM
);
352 const MCSubtargetInfo
*MCInfo
= Target
->getMCSubtargetInfo();
353 const ArrayRef
<SubtargetFeatureKV
> FeatTable
= MCInfo
->getFeatureTable();
354 const SubtargetFeatureKV Feat
= FeatTable
[Index
];
361 extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef
) {
362 printf("Target CPU help is not supported by this LLVM version.\n\n");
365 extern "C" size_t LLVMRustGetTargetFeaturesCount(LLVMTargetMachineRef
) {
369 extern "C" void LLVMRustGetTargetFeature(LLVMTargetMachineRef
, const char**, const char**) {}
372 extern "C" const char* LLVMRustGetHostCPUName(size_t *len
) {
373 StringRef Name
= sys::getHostCPUName();
378 extern "C" LLVMTargetMachineRef
LLVMRustCreateTargetMachine(
379 const char *TripleStr
, const char *CPU
, const char *Feature
,
380 const char *ABIStr
, LLVMRustCodeModel RustCM
, LLVMRustRelocModel RustReloc
,
381 LLVMRustCodeGenOptLevel RustOptLevel
, bool UseSoftFloat
,
382 bool FunctionSections
,
384 bool UniqueSectionNames
,
385 bool TrapUnreachable
,
388 bool EmitStackSizeSection
,
389 bool RelaxELFRelocations
,
391 const char *SplitDwarfFile
) {
393 auto OptLevel
= fromRust(RustOptLevel
);
394 auto RM
= fromRust(RustReloc
);
395 auto CM
= fromRust(RustCM
);
398 Triple
Trip(Triple::normalize(TripleStr
));
399 const llvm::Target
*TheTarget
=
400 TargetRegistry::lookupTarget(Trip
.getTriple(), Error
);
401 if (TheTarget
== nullptr) {
402 LLVMRustSetLastError(Error
.c_str());
406 TargetOptions Options
;
408 Options
.FloatABIType
= FloatABI::Default
;
410 Options
.FloatABIType
= FloatABI::Soft
;
412 Options
.DataSections
= DataSections
;
413 Options
.FunctionSections
= FunctionSections
;
414 Options
.UniqueSectionNames
= UniqueSectionNames
;
415 Options
.MCOptions
.AsmVerbose
= AsmComments
;
416 Options
.MCOptions
.PreserveAsmComments
= AsmComments
;
417 Options
.MCOptions
.ABIName
= ABIStr
;
418 if (SplitDwarfFile
) {
419 Options
.MCOptions
.SplitDwarfFile
= SplitDwarfFile
;
421 Options
.RelaxELFRelocations
= RelaxELFRelocations
;
422 Options
.UseInitArray
= UseInitArray
;
424 if (TrapUnreachable
) {
425 // Tell LLVM to codegen `unreachable` into an explicit trap instruction.
426 // This limits the extent of possible undefined behavior in some cases, as
427 // it prevents control flow from "falling through" into whatever code
428 // happens to be laid out next in memory.
429 Options
.TrapUnreachable
= true;
433 Options
.ThreadModel
= ThreadModel::Single
;
436 Options
.EmitStackSizeSection
= EmitStackSizeSection
;
438 TargetMachine
*TM
= TheTarget
->createTargetMachine(
439 Trip
.getTriple(), CPU
, Feature
, Options
, RM
, CM
, OptLevel
);
443 extern "C" void LLVMRustDisposeTargetMachine(LLVMTargetMachineRef TM
) {
447 // Unfortunately, the LLVM C API doesn't provide a way to create the
448 // TargetLibraryInfo pass, so we use this method to do so.
449 extern "C" void LLVMRustAddLibraryInfo(LLVMPassManagerRef PMR
, LLVMModuleRef M
,
450 bool DisableSimplifyLibCalls
) {
451 Triple
TargetTriple(unwrap(M
)->getTargetTriple());
452 TargetLibraryInfoImpl
TLII(TargetTriple
);
453 if (DisableSimplifyLibCalls
)
454 TLII
.disableAllFunctions();
455 unwrap(PMR
)->add(new TargetLibraryInfoWrapperPass(TLII
));
458 extern "C" void LLVMRustSetLLVMOptions(int Argc
, char **Argv
) {
459 // Initializing the command-line options more than once is not allowed. So,
460 // check if they've already been initialized. (This could happen if we're
461 // being called from rustpkg, for example). If the arguments change, then
462 // that's just kinda unfortunate.
463 static bool Initialized
= false;
467 cl::ParseCommandLineOptions(Argc
, Argv
);
470 enum class LLVMRustFileType
{
475 static CodeGenFileType
fromRust(LLVMRustFileType Type
) {
477 case LLVMRustFileType::AssemblyFile
:
478 return CGFT_AssemblyFile
;
479 case LLVMRustFileType::ObjectFile
:
480 return CGFT_ObjectFile
;
482 report_fatal_error("Bad FileType.");
486 extern "C" LLVMRustResult
487 LLVMRustWriteOutputFile(LLVMTargetMachineRef Target
, LLVMPassManagerRef PMR
,
488 LLVMModuleRef M
, const char *Path
, const char *DwoPath
,
489 LLVMRustFileType RustFileType
) {
490 llvm::legacy::PassManager
*PM
= unwrap
<llvm::legacy::PassManager
>(PMR
);
491 auto FileType
= fromRust(RustFileType
);
493 std::string ErrorInfo
;
495 raw_fd_ostream
OS(Path
, EC
, sys::fs::OF_None
);
497 ErrorInfo
= EC
.message();
498 if (ErrorInfo
!= "") {
499 LLVMRustSetLastError(ErrorInfo
.c_str());
500 return LLVMRustResult::Failure
;
503 buffer_ostream
BOS(OS
);
505 raw_fd_ostream
DOS(DwoPath
, EC
, sys::fs::OF_None
);
508 ErrorInfo
= EC
.message();
509 if (ErrorInfo
!= "") {
510 LLVMRustSetLastError(ErrorInfo
.c_str());
511 return LLVMRustResult::Failure
;
513 buffer_ostream
DBOS(DOS
);
514 unwrap(Target
)->addPassesToEmitFile(*PM
, BOS
, &DBOS
, FileType
, false);
517 unwrap(Target
)->addPassesToEmitFile(*PM
, BOS
, nullptr, FileType
, false);
521 // Apparently `addPassesToEmitFile` adds a pointer to our on-the-stack output
522 // stream (OS), so the only real safe place to delete this is here? Don't we
523 // wish this was written in Rust?
524 LLVMDisposePassManager(PMR
);
525 return LLVMRustResult::Success
;
528 extern "C" typedef void (*LLVMRustSelfProfileBeforePassCallback
)(void*, // LlvmSelfProfiler
529 const char*, // pass name
530 const char*); // IR name
531 extern "C" typedef void (*LLVMRustSelfProfileAfterPassCallback
)(void*); // LlvmSelfProfiler
533 std::string
LLVMRustwrappedIrGetName(const llvm::Any
&WrappedIr
) {
534 if (any_isa
<const Module
*>(WrappedIr
))
535 return any_cast
<const Module
*>(WrappedIr
)->getName().str();
536 if (any_isa
<const Function
*>(WrappedIr
))
537 return any_cast
<const Function
*>(WrappedIr
)->getName().str();
538 if (any_isa
<const Loop
*>(WrappedIr
))
539 return any_cast
<const Loop
*>(WrappedIr
)->getName().str();
540 if (any_isa
<const LazyCallGraph::SCC
*>(WrappedIr
))
541 return any_cast
<const LazyCallGraph::SCC
*>(WrappedIr
)->getName();
546 void LLVMSelfProfileInitializeCallbacks(
547 PassInstrumentationCallbacks
& PIC
, void* LlvmSelfProfiler
,
548 LLVMRustSelfProfileBeforePassCallback BeforePassCallback
,
549 LLVMRustSelfProfileAfterPassCallback AfterPassCallback
) {
550 PIC
.registerBeforeNonSkippedPassCallback([LlvmSelfProfiler
, BeforePassCallback
](
551 StringRef Pass
, llvm::Any Ir
) {
552 std::string PassName
= Pass
.str();
553 std::string IrName
= LLVMRustwrappedIrGetName(Ir
);
554 BeforePassCallback(LlvmSelfProfiler
, PassName
.c_str(), IrName
.c_str());
557 PIC
.registerAfterPassCallback(
558 [LlvmSelfProfiler
, AfterPassCallback
](StringRef Pass
, llvm::Any IR
,
559 const PreservedAnalyses
&Preserved
) {
560 AfterPassCallback(LlvmSelfProfiler
);
563 PIC
.registerAfterPassInvalidatedCallback(
564 [LlvmSelfProfiler
, AfterPassCallback
](StringRef Pass
, const PreservedAnalyses
&Preserved
) {
565 AfterPassCallback(LlvmSelfProfiler
);
568 PIC
.registerBeforeAnalysisCallback([LlvmSelfProfiler
, BeforePassCallback
](
569 StringRef Pass
, llvm::Any Ir
) {
570 std::string PassName
= Pass
.str();
571 std::string IrName
= LLVMRustwrappedIrGetName(Ir
);
572 BeforePassCallback(LlvmSelfProfiler
, PassName
.c_str(), IrName
.c_str());
575 PIC
.registerAfterAnalysisCallback(
576 [LlvmSelfProfiler
, AfterPassCallback
](StringRef Pass
, llvm::Any Ir
) {
577 AfterPassCallback(LlvmSelfProfiler
);
581 enum class LLVMRustOptStage
{
589 struct LLVMRustSanitizerOptions
{
590 bool SanitizeAddress
;
591 bool SanitizeAddressRecover
;
593 bool SanitizeMemoryRecover
;
594 int SanitizeMemoryTrackOrigins
;
596 bool SanitizeHWAddress
;
597 bool SanitizeHWAddressRecover
;
600 extern "C" LLVMRustResult
602 LLVMModuleRef ModuleRef
,
603 LLVMTargetMachineRef TMRef
,
604 LLVMRustPassBuilderOptLevel OptLevelRust
,
605 LLVMRustOptStage OptStage
,
606 bool NoPrepopulatePasses
, bool VerifyIR
, bool UseThinLTOBuffers
,
607 bool MergeFunctions
, bool UnrollLoops
, bool SLPVectorize
, bool LoopVectorize
,
608 bool DisableSimplifyLibCalls
, bool EmitLifetimeMarkers
,
609 LLVMRustSanitizerOptions
*SanitizerOptions
,
610 const char *PGOGenPath
, const char *PGOUsePath
,
611 bool InstrumentCoverage
, const char *InstrProfileOutput
,
613 const char *PGOSampleUsePath
, bool DebugInfoForProfiling
,
614 void* LlvmSelfProfiler
,
615 LLVMRustSelfProfileBeforePassCallback BeforePassCallback
,
616 LLVMRustSelfProfileAfterPassCallback AfterPassCallback
,
617 const char *ExtraPasses
, size_t ExtraPassesLen
,
618 const char *LLVMPlugins
, size_t LLVMPluginsLen
) {
619 Module
*TheModule
= unwrap(ModuleRef
);
620 TargetMachine
*TM
= unwrap(TMRef
);
621 OptimizationLevel OptLevel
= fromRust(OptLevelRust
);
624 PipelineTuningOptions PTO
;
625 PTO
.LoopUnrolling
= UnrollLoops
;
626 PTO
.LoopInterleaving
= UnrollLoops
;
627 PTO
.LoopVectorization
= LoopVectorize
;
628 PTO
.SLPVectorization
= SLPVectorize
;
629 PTO
.MergeFunctions
= MergeFunctions
;
631 // FIXME: We may want to expose this as an option.
632 bool DebugPassManager
= false;
634 PassInstrumentationCallbacks PIC
;
635 #if LLVM_VERSION_LT(16, 0)
636 StandardInstrumentations
SI(DebugPassManager
);
638 StandardInstrumentations
SI(TheModule
->getContext(), DebugPassManager
);
640 SI
.registerCallbacks(PIC
);
642 if (LlvmSelfProfiler
){
643 LLVMSelfProfileInitializeCallbacks(PIC
,LlvmSelfProfiler
,BeforePassCallback
,AfterPassCallback
);
646 #if LLVM_VERSION_LT(16, 0)
647 Optional
<PGOOptions
> PGOOpt
;
649 std::optional
<PGOOptions
> PGOOpt
;
652 assert(!PGOUsePath
&& !PGOSampleUsePath
);
653 PGOOpt
= PGOOptions(PGOGenPath
, "", "", PGOOptions::IRInstr
,
654 PGOOptions::NoCSAction
, DebugInfoForProfiling
);
655 } else if (PGOUsePath
) {
656 assert(!PGOSampleUsePath
);
657 PGOOpt
= PGOOptions(PGOUsePath
, "", "", PGOOptions::IRUse
,
658 PGOOptions::NoCSAction
, DebugInfoForProfiling
);
659 } else if (PGOSampleUsePath
) {
660 PGOOpt
= PGOOptions(PGOSampleUsePath
, "", "", PGOOptions::SampleUse
,
661 PGOOptions::NoCSAction
, DebugInfoForProfiling
);
662 } else if (DebugInfoForProfiling
) {
663 PGOOpt
= PGOOptions("", "", "", PGOOptions::NoAction
,
664 PGOOptions::NoCSAction
, DebugInfoForProfiling
);
667 PassBuilder
PB(TM
, PTO
, PGOOpt
, &PIC
);
668 LoopAnalysisManager LAM
;
669 FunctionAnalysisManager FAM
;
670 CGSCCAnalysisManager CGAM
;
671 ModuleAnalysisManager MAM
;
673 FAM
.registerPass([&] { return PB
.buildDefaultAAPipeline(); });
675 Triple
TargetTriple(TheModule
->getTargetTriple());
676 std::unique_ptr
<TargetLibraryInfoImpl
> TLII(new TargetLibraryInfoImpl(TargetTriple
));
677 if (DisableSimplifyLibCalls
)
678 TLII
->disableAllFunctions();
679 FAM
.registerPass([&] { return TargetLibraryAnalysis(*TLII
); });
681 PB
.registerModuleAnalyses(MAM
);
682 PB
.registerCGSCCAnalyses(CGAM
);
683 PB
.registerFunctionAnalyses(FAM
);
684 PB
.registerLoopAnalyses(LAM
);
685 PB
.crossRegisterProxies(LAM
, FAM
, CGAM
, MAM
);
687 // We manually collect pipeline callbacks so we can apply them at O0, where the
688 // PassBuilder does not create a pipeline.
689 std::vector
<std::function
<void(ModulePassManager
&, OptimizationLevel
)>>
690 PipelineStartEPCallbacks
;
691 std::vector
<std::function
<void(ModulePassManager
&, OptimizationLevel
)>>
692 OptimizerLastEPCallbacks
;
695 PipelineStartEPCallbacks
.push_back(
696 [VerifyIR
](ModulePassManager
&MPM
, OptimizationLevel Level
) {
697 MPM
.addPass(VerifierPass());
702 if (InstrumentGCOV
) {
703 PipelineStartEPCallbacks
.push_back(
704 [](ModulePassManager
&MPM
, OptimizationLevel Level
) {
705 MPM
.addPass(GCOVProfilerPass(GCOVOptions::getDefault()));
710 if (InstrumentCoverage
) {
711 PipelineStartEPCallbacks
.push_back(
712 [InstrProfileOutput
](ModulePassManager
&MPM
, OptimizationLevel Level
) {
713 InstrProfOptions Options
;
714 if (InstrProfileOutput
) {
715 Options
.InstrProfileOutput
= InstrProfileOutput
;
717 MPM
.addPass(InstrProfiling(Options
, false));
722 if (SanitizerOptions
) {
723 if (SanitizerOptions
->SanitizeMemory
) {
724 #if LLVM_VERSION_GE(14, 0)
725 MemorySanitizerOptions
Options(
726 SanitizerOptions
->SanitizeMemoryTrackOrigins
,
727 SanitizerOptions
->SanitizeMemoryRecover
,
728 /*CompileKernel=*/false,
729 /*EagerChecks=*/true);
731 MemorySanitizerOptions
Options(
732 SanitizerOptions
->SanitizeMemoryTrackOrigins
,
733 SanitizerOptions
->SanitizeMemoryRecover
,
734 /*CompileKernel=*/false);
736 OptimizerLastEPCallbacks
.push_back(
737 [Options
](ModulePassManager
&MPM
, OptimizationLevel Level
) {
738 #if LLVM_VERSION_GE(14, 0) && LLVM_VERSION_LT(16, 0)
739 MPM
.addPass(ModuleMemorySanitizerPass(Options
));
741 MPM
.addPass(MemorySanitizerPass(Options
));
743 #if LLVM_VERSION_LT(16, 0)
744 MPM
.addPass(createModuleToFunctionPassAdaptor(MemorySanitizerPass(Options
)));
750 if (SanitizerOptions
->SanitizeThread
) {
751 OptimizerLastEPCallbacks
.push_back(
752 [](ModulePassManager
&MPM
, OptimizationLevel Level
) {
753 #if LLVM_VERSION_GE(14, 0)
754 MPM
.addPass(ModuleThreadSanitizerPass());
756 MPM
.addPass(ThreadSanitizerPass());
758 MPM
.addPass(createModuleToFunctionPassAdaptor(ThreadSanitizerPass()));
763 if (SanitizerOptions
->SanitizeAddress
) {
764 OptimizerLastEPCallbacks
.push_back(
765 [SanitizerOptions
](ModulePassManager
&MPM
, OptimizationLevel Level
) {
766 #if LLVM_VERSION_LT(15, 0)
767 MPM
.addPass(RequireAnalysisPass
<ASanGlobalsMetadataAnalysis
, Module
>());
769 #if LLVM_VERSION_GE(14, 0)
770 AddressSanitizerOptions opts
= AddressSanitizerOptions
{
771 /*CompileKernel=*/false,
772 SanitizerOptions
->SanitizeAddressRecover
,
773 /*UseAfterScope=*/true,
774 AsanDetectStackUseAfterReturnMode::Runtime
,
776 #if LLVM_VERSION_LT(16, 0)
777 MPM
.addPass(ModuleAddressSanitizerPass(opts
));
779 MPM
.addPass(AddressSanitizerPass(opts
));
782 MPM
.addPass(ModuleAddressSanitizerPass(
783 /*CompileKernel=*/false, SanitizerOptions
->SanitizeAddressRecover
));
784 MPM
.addPass(createModuleToFunctionPassAdaptor(AddressSanitizerPass(
785 /*CompileKernel=*/false, SanitizerOptions
->SanitizeAddressRecover
,
786 /*UseAfterScope=*/true)));
791 if (SanitizerOptions
->SanitizeHWAddress
) {
792 OptimizerLastEPCallbacks
.push_back(
793 [SanitizerOptions
](ModulePassManager
&MPM
, OptimizationLevel Level
) {
794 #if LLVM_VERSION_GE(14, 0)
795 HWAddressSanitizerOptions
opts(
796 /*CompileKernel=*/false, SanitizerOptions
->SanitizeHWAddressRecover
,
797 /*DisableOptimization=*/false);
798 MPM
.addPass(HWAddressSanitizerPass(opts
));
800 MPM
.addPass(HWAddressSanitizerPass(
801 /*CompileKernel=*/false, SanitizerOptions
->SanitizeHWAddressRecover
));
808 if (LLVMPluginsLen
) {
809 auto PluginsStr
= StringRef(LLVMPlugins
, LLVMPluginsLen
);
810 SmallVector
<StringRef
> Plugins
;
811 PluginsStr
.split(Plugins
, ',', -1, false);
812 for (auto PluginPath
: Plugins
) {
813 auto Plugin
= PassPlugin::Load(PluginPath
.str());
815 LLVMRustSetLastError(("Failed to load pass plugin" + PluginPath
.str()).c_str());
816 return LLVMRustResult::Failure
;
818 Plugin
->registerPassBuilderCallbacks(PB
);
822 ModulePassManager MPM
;
823 bool NeedThinLTOBufferPasses
= UseThinLTOBuffers
;
824 if (!NoPrepopulatePasses
) {
825 // The pre-link pipelines don't support O0 and require using budilO0DefaultPipeline() instead.
826 // At the same time, the LTO pipelines do support O0 and using them is required.
827 bool IsLTO
= OptStage
== LLVMRustOptStage::ThinLTO
|| OptStage
== LLVMRustOptStage::FatLTO
;
828 if (OptLevel
== OptimizationLevel::O0
&& !IsLTO
) {
829 for (const auto &C
: PipelineStartEPCallbacks
)
830 PB
.registerPipelineStartEPCallback(C
);
831 for (const auto &C
: OptimizerLastEPCallbacks
)
832 PB
.registerOptimizerLastEPCallback(C
);
834 // Pass false as we manually schedule ThinLTOBufferPasses below.
835 MPM
= PB
.buildO0DefaultPipeline(OptLevel
, /* PreLinkLTO */ false);
837 for (const auto &C
: PipelineStartEPCallbacks
)
838 PB
.registerPipelineStartEPCallback(C
);
839 if (OptStage
!= LLVMRustOptStage::PreLinkThinLTO
) {
840 for (const auto &C
: OptimizerLastEPCallbacks
)
841 PB
.registerOptimizerLastEPCallback(C
);
845 case LLVMRustOptStage::PreLinkNoLTO
:
846 MPM
= PB
.buildPerModuleDefaultPipeline(OptLevel
, DebugPassManager
);
848 case LLVMRustOptStage::PreLinkThinLTO
:
849 MPM
= PB
.buildThinLTOPreLinkDefaultPipeline(OptLevel
);
850 // The ThinLTOPreLink pipeline already includes ThinLTOBuffer passes. However, callback
851 // passes may still run afterwards. This means we need to run the buffer passes again.
852 // FIXME: In LLVM 13, the ThinLTOPreLink pipeline also runs OptimizerLastEPCallbacks
853 // before the RequiredLTOPreLinkPasses, in which case we can remove these hacks.
854 if (OptimizerLastEPCallbacks
.empty())
855 NeedThinLTOBufferPasses
= false;
856 for (const auto &C
: OptimizerLastEPCallbacks
)
859 case LLVMRustOptStage::PreLinkFatLTO
:
860 MPM
= PB
.buildLTOPreLinkDefaultPipeline(OptLevel
);
861 NeedThinLTOBufferPasses
= false;
863 case LLVMRustOptStage::ThinLTO
:
864 // FIXME: Does it make sense to pass the ModuleSummaryIndex?
865 // It only seems to be needed for C++ specific optimizations.
866 MPM
= PB
.buildThinLTODefaultPipeline(OptLevel
, nullptr);
868 case LLVMRustOptStage::FatLTO
:
869 MPM
= PB
.buildLTODefaultPipeline(OptLevel
, nullptr);
874 // We're not building any of the default pipelines but we still want to
875 // add the verifier, instrumentation, etc passes if they were requested
876 for (const auto &C
: PipelineStartEPCallbacks
)
878 for (const auto &C
: OptimizerLastEPCallbacks
)
882 if (ExtraPassesLen
) {
883 if (auto Err
= PB
.parsePassPipeline(MPM
, StringRef(ExtraPasses
, ExtraPassesLen
))) {
884 std::string ErrMsg
= toString(std::move(Err
));
885 LLVMRustSetLastError(ErrMsg
.c_str());
886 return LLVMRustResult::Failure
;
890 if (NeedThinLTOBufferPasses
) {
891 MPM
.addPass(CanonicalizeAliasesPass());
892 MPM
.addPass(NameAnonGlobalPass());
895 // Upgrade all calls to old intrinsics first.
896 for (Module::iterator I
= TheModule
->begin(), E
= TheModule
->end(); I
!= E
;)
897 UpgradeCallsToIntrinsic(&*I
++); // must be post-increment, as we remove
899 MPM
.run(*TheModule
, MAM
);
900 return LLVMRustResult::Success
;
903 // Callback to demangle function name
905 // * name to be demangled
908 // * output buffer len
909 // Returns len of demangled string, or 0 if demangle failed.
910 typedef size_t (*DemangleFn
)(const char*, size_t, char*, size_t);
915 class RustAssemblyAnnotationWriter
: public AssemblyAnnotationWriter
{
917 std::vector
<char> Buf
;
920 RustAssemblyAnnotationWriter(DemangleFn Demangle
) : Demangle(Demangle
) {}
922 // Return empty string if demangle failed
923 // or if name does not need to be demangled
924 StringRef
CallDemangle(StringRef name
) {
929 if (Buf
.size() < name
.size() * 2) {
930 // Semangled name usually shorter than mangled,
931 // but allocate twice as much memory just in case
932 Buf
.resize(name
.size() * 2);
935 auto R
= Demangle(name
.data(), name
.size(), Buf
.data(), Buf
.size());
941 auto Demangled
= StringRef(Buf
.data(), R
);
942 if (Demangled
== name
) {
943 // Do not print anything if demangled name is equal to mangled.
950 void emitFunctionAnnot(const Function
*F
,
951 formatted_raw_ostream
&OS
) override
{
952 StringRef Demangled
= CallDemangle(F
->getName());
953 if (Demangled
.empty()) {
957 OS
<< "; " << Demangled
<< "\n";
960 void emitInstructionAnnot(const Instruction
*I
,
961 formatted_raw_ostream
&OS
) override
{
964 if (const CallInst
*CI
= dyn_cast
<CallInst
>(I
)) {
966 Value
= CI
->getCalledOperand();
967 } else if (const InvokeInst
* II
= dyn_cast
<InvokeInst
>(I
)) {
969 Value
= II
->getCalledOperand();
971 // Could demangle more operations, e. g.
972 // `store %place, @function`.
976 if (!Value
->hasName()) {
980 StringRef Demangled
= CallDemangle(Value
->getName());
981 if (Demangled
.empty()) {
985 OS
<< "; " << Name
<< " " << Demangled
<< "\n";
991 extern "C" LLVMRustResult
992 LLVMRustPrintModule(LLVMModuleRef M
, const char *Path
, DemangleFn Demangle
) {
993 std::string ErrorInfo
;
995 raw_fd_ostream
OS(Path
, EC
, sys::fs::OF_None
);
997 ErrorInfo
= EC
.message();
998 if (ErrorInfo
!= "") {
999 LLVMRustSetLastError(ErrorInfo
.c_str());
1000 return LLVMRustResult::Failure
;
1003 RustAssemblyAnnotationWriter
AAW(Demangle
);
1004 formatted_raw_ostream
FOS(OS
);
1005 unwrap(M
)->print(FOS
, &AAW
);
1007 return LLVMRustResult::Success
;
1010 extern "C" void LLVMRustPrintPasses() {
1011 LLVMInitializePasses();
1012 struct MyListener
: PassRegistrationListener
{
1013 void passEnumerate(const PassInfo
*Info
) {
1014 StringRef PassArg
= Info
->getPassArgument();
1015 StringRef PassName
= Info
->getPassName();
1016 if (!PassArg
.empty()) {
1017 // These unsigned->signed casts could theoretically overflow, but
1018 // realistically never will (and even if, the result is implementation
1019 // defined rather plain UB).
1020 printf("%15.*s - %.*s\n", (int)PassArg
.size(), PassArg
.data(),
1021 (int)PassName
.size(), PassName
.data());
1026 PassRegistry
*PR
= PassRegistry::getPassRegistry();
1027 PR
->enumerateWith(&Listener
);
1030 extern "C" void LLVMRustRunRestrictionPass(LLVMModuleRef M
, char **Symbols
,
1032 auto PreserveFunctions
= [=](const GlobalValue
&GV
) {
1033 for (size_t I
= 0; I
< Len
; I
++) {
1034 if (GV
.getName() == Symbols
[I
]) {
1041 internalizeModule(*unwrap(M
), PreserveFunctions
);
1045 LLVMRustSetDataLayoutFromTargetMachine(LLVMModuleRef Module
,
1046 LLVMTargetMachineRef TMR
) {
1047 TargetMachine
*Target
= unwrap(TMR
);
1048 unwrap(Module
)->setDataLayout(Target
->createDataLayout());
1051 extern "C" void LLVMRustSetModulePICLevel(LLVMModuleRef M
) {
1052 unwrap(M
)->setPICLevel(PICLevel::Level::BigPIC
);
1055 extern "C" void LLVMRustSetModulePIELevel(LLVMModuleRef M
) {
1056 unwrap(M
)->setPIELevel(PIELevel::Level::Large
);
1059 extern "C" void LLVMRustSetModuleCodeModel(LLVMModuleRef M
,
1060 LLVMRustCodeModel Model
) {
1061 auto CM
= fromRust(Model
);
1064 unwrap(M
)->setCodeModel(*CM
);
1067 // Here you'll find an implementation of ThinLTO as used by the Rust compiler
1068 // right now. This ThinLTO support is only enabled on "recent ish" versions of
1069 // LLVM, and otherwise it's just blanket rejected from other compilers.
1071 // Most of this implementation is straight copied from LLVM. At the time of
1072 // this writing it wasn't *quite* suitable to reuse more code from upstream
1073 // for our purposes, but we should strive to upstream this support once it's
1074 // ready to go! I figure we may want a bit of testing locally first before
1075 // sending this upstream to LLVM. I hear though they're quite eager to receive
1076 // feedback like this!
1078 // If you're reading this code and wondering "what in the world" or you're
1079 // working "good lord by LLVM upgrade is *still* failing due to these bindings"
1080 // then fear not! (ok maybe fear a little). All code here is mostly based
1081 // on `lib/LTO/ThinLTOCodeGenerator.cpp` in LLVM.
1083 // You'll find that the general layout here roughly corresponds to the `run`
1084 // method in that file as well as `ProcessThinLTOModule`. Functions are
1085 // specifically commented below as well, but if you're updating this code
1086 // or otherwise trying to understand it, the LLVM source will be useful in
1087 // interpreting the mysteries within.
1089 // Otherwise I'll apologize in advance, it probably requires a relatively
1090 // significant investment on your part to "truly understand" what's going on
1091 // here. Not saying I do myself, but it took me awhile staring at LLVM's source
1092 // and various online resources about ThinLTO to make heads or tails of all
1095 // This is a shared data structure which *must* be threadsafe to share
1096 // read-only amongst threads. This also corresponds basically to the arguments
1097 // of the `ProcessThinLTOModule` function in the LLVM source.
1098 struct LLVMRustThinLTOData
{
1099 // The combined index that is the global analysis over all modules we're
1100 // performing ThinLTO for. This is mostly managed by LLVM.
1101 ModuleSummaryIndex Index
;
1103 // All modules we may look at, stored as in-memory serialized versions. This
1104 // is later used when inlining to ensure we can extract any module to inline
1106 StringMap
<MemoryBufferRef
> ModuleMap
;
1108 // A set that we manage of everything we *don't* want internalized. Note that
1109 // this includes all transitive references right now as well, but it may not
1111 DenseSet
<GlobalValue::GUID
> GUIDPreservedSymbols
;
1113 // Not 100% sure what these are, but they impact what's internalized and
1114 // what's inlined across modules, I believe.
1115 StringMap
<FunctionImporter::ImportMapTy
> ImportLists
;
1116 StringMap
<FunctionImporter::ExportSetTy
> ExportLists
;
1117 StringMap
<GVSummaryMapTy
> ModuleToDefinedGVSummaries
;
1118 StringMap
<std::map
<GlobalValue::GUID
, GlobalValue::LinkageTypes
>> ResolvedODR
;
1120 LLVMRustThinLTOData() : Index(/* HaveGVs = */ false) {}
1123 // Just an argument to the `LLVMRustCreateThinLTOData` function below.
1124 struct LLVMRustThinLTOModule
{
1125 const char *identifier
;
1130 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`, not sure what it
1132 static const GlobalValueSummary
*
1133 getFirstDefinitionForLinker(const GlobalValueSummaryList
&GVSummaryList
) {
1134 auto StrongDefForLinker
= llvm::find_if(
1135 GVSummaryList
, [](const std::unique_ptr
<GlobalValueSummary
> &Summary
) {
1136 auto Linkage
= Summary
->linkage();
1137 return !GlobalValue::isAvailableExternallyLinkage(Linkage
) &&
1138 !GlobalValue::isWeakForLinker(Linkage
);
1140 if (StrongDefForLinker
!= GVSummaryList
.end())
1141 return StrongDefForLinker
->get();
1143 auto FirstDefForLinker
= llvm::find_if(
1144 GVSummaryList
, [](const std::unique_ptr
<GlobalValueSummary
> &Summary
) {
1145 auto Linkage
= Summary
->linkage();
1146 return !GlobalValue::isAvailableExternallyLinkage(Linkage
);
1148 if (FirstDefForLinker
== GVSummaryList
.end())
1150 return FirstDefForLinker
->get();
1153 // The main entry point for creating the global ThinLTO analysis. The structure
1154 // here is basically the same as before threads are spawned in the `run`
1155 // function of `lib/LTO/ThinLTOCodeGenerator.cpp`.
1156 extern "C" LLVMRustThinLTOData
*
1157 LLVMRustCreateThinLTOData(LLVMRustThinLTOModule
*modules
,
1159 const char **preserved_symbols
,
1161 auto Ret
= std::make_unique
<LLVMRustThinLTOData
>();
1163 // Load each module's summary and merge it into one combined index
1164 for (int i
= 0; i
< num_modules
; i
++) {
1165 auto module
= &modules
[i
];
1166 StringRef
buffer(module
->data
, module
->len
);
1167 MemoryBufferRef
mem_buffer(buffer
, module
->identifier
);
1169 Ret
->ModuleMap
[module
->identifier
] = mem_buffer
;
1171 if (Error Err
= readModuleSummaryIndex(mem_buffer
, Ret
->Index
, i
)) {
1172 LLVMRustSetLastError(toString(std::move(Err
)).c_str());
1177 // Collect for each module the list of function it defines (GUID -> Summary)
1178 Ret
->Index
.collectDefinedGVSummariesPerModule(Ret
->ModuleToDefinedGVSummaries
);
1180 // Convert the preserved symbols set from string to GUID, this is then needed
1181 // for internalization.
1182 for (int i
= 0; i
< num_symbols
; i
++) {
1183 auto GUID
= GlobalValue::getGUID(preserved_symbols
[i
]);
1184 Ret
->GUIDPreservedSymbols
.insert(GUID
);
1187 // Collect the import/export lists for all modules from the call-graph in the
1190 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`
1191 auto deadIsPrevailing
= [&](GlobalValue::GUID G
) {
1192 return PrevailingType::Unknown
;
1194 // We don't have a complete picture in our use of ThinLTO, just our immediate
1195 // crate, so we need `ImportEnabled = false` to limit internalization.
1196 // Otherwise, we sometimes lose `static` values -- see #60184.
1197 computeDeadSymbolsWithConstProp(Ret
->Index
, Ret
->GUIDPreservedSymbols
,
1198 deadIsPrevailing
, /* ImportEnabled = */ false);
1199 ComputeCrossModuleImport(
1201 Ret
->ModuleToDefinedGVSummaries
,
1206 // Resolve LinkOnce/Weak symbols, this has to be computed early be cause it
1207 // impacts the caching.
1209 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` with some of this
1210 // being lifted from `lib/LTO/LTO.cpp` as well
1211 DenseMap
<GlobalValue::GUID
, const GlobalValueSummary
*> PrevailingCopy
;
1212 for (auto &I
: Ret
->Index
) {
1213 if (I
.second
.SummaryList
.size() > 1)
1214 PrevailingCopy
[I
.first
] = getFirstDefinitionForLinker(I
.second
.SummaryList
);
1216 auto isPrevailing
= [&](GlobalValue::GUID GUID
, const GlobalValueSummary
*S
) {
1217 const auto &Prevailing
= PrevailingCopy
.find(GUID
);
1218 if (Prevailing
== PrevailingCopy
.end())
1220 return Prevailing
->second
== S
;
1222 auto recordNewLinkage
= [&](StringRef ModuleIdentifier
,
1223 GlobalValue::GUID GUID
,
1224 GlobalValue::LinkageTypes NewLinkage
) {
1225 Ret
->ResolvedODR
[ModuleIdentifier
][GUID
] = NewLinkage
;
1228 // Uses FromPrevailing visibility scheme which works for many binary
1229 // formats. We probably could and should use ELF visibility scheme for many of
1230 // our targets, however.
1232 thinLTOResolvePrevailingInIndex(conf
, Ret
->Index
, isPrevailing
, recordNewLinkage
,
1233 Ret
->GUIDPreservedSymbols
);
1235 // Here we calculate an `ExportedGUIDs` set for use in the `isExported`
1236 // callback below. This callback below will dictate the linkage for all
1237 // summaries in the index, and we basically just only want to ensure that dead
1238 // symbols are internalized. Otherwise everything that's already external
1239 // linkage will stay as external, and internal will stay as internal.
1240 std::set
<GlobalValue::GUID
> ExportedGUIDs
;
1241 for (auto &List
: Ret
->Index
) {
1242 for (auto &GVS
: List
.second
.SummaryList
) {
1243 if (GlobalValue::isLocalLinkage(GVS
->linkage()))
1245 auto GUID
= GVS
->getOriginalName();
1246 if (GVS
->flags().Live
)
1247 ExportedGUIDs
.insert(GUID
);
1250 auto isExported
= [&](StringRef ModuleIdentifier
, ValueInfo VI
) {
1251 const auto &ExportList
= Ret
->ExportLists
.find(ModuleIdentifier
);
1252 return (ExportList
!= Ret
->ExportLists
.end() &&
1253 ExportList
->second
.count(VI
)) ||
1254 ExportedGUIDs
.count(VI
.getGUID());
1256 thinLTOInternalizeAndPromoteInIndex(Ret
->Index
, isExported
, isPrevailing
);
1258 return Ret
.release();
1262 LLVMRustFreeThinLTOData(LLVMRustThinLTOData
*Data
) {
1266 // Below are the various passes that happen *per module* when doing ThinLTO.
1268 // In other words, these are the functions that are all run concurrently
1269 // with one another, one per module. The passes here correspond to the analysis
1270 // passes in `lib/LTO/ThinLTOCodeGenerator.cpp`, currently found in the
1271 // `ProcessThinLTOModule` function. Here they're split up into separate steps
1272 // so rustc can save off the intermediate bytecode between each step.
1275 clearDSOLocalOnDeclarations(Module
&Mod
, TargetMachine
&TM
) {
1276 // When linking an ELF shared object, dso_local should be dropped. We
1277 // conservatively do this for -fpic.
1278 bool ClearDSOLocalOnDeclarations
=
1279 TM
.getTargetTriple().isOSBinFormatELF() &&
1280 TM
.getRelocationModel() != Reloc::Static
&&
1281 Mod
.getPIELevel() == PIELevel::Default
;
1282 return ClearDSOLocalOnDeclarations
;
1286 LLVMRustPrepareThinLTORename(const LLVMRustThinLTOData
*Data
, LLVMModuleRef M
,
1287 LLVMTargetMachineRef TM
) {
1288 Module
&Mod
= *unwrap(M
);
1289 TargetMachine
&Target
= *unwrap(TM
);
1291 bool ClearDSOLocal
= clearDSOLocalOnDeclarations(Mod
, Target
);
1292 bool error
= renameModuleForThinLTO(Mod
, Data
->Index
, ClearDSOLocal
);
1295 LLVMRustSetLastError("renameModuleForThinLTO failed");
1302 LLVMRustPrepareThinLTOResolveWeak(const LLVMRustThinLTOData
*Data
, LLVMModuleRef M
) {
1303 Module
&Mod
= *unwrap(M
);
1304 const auto &DefinedGlobals
= Data
->ModuleToDefinedGVSummaries
.lookup(Mod
.getModuleIdentifier());
1305 #if LLVM_VERSION_GE(14, 0)
1306 thinLTOFinalizeInModule(Mod
, DefinedGlobals
, /*PropagateAttrs=*/true);
1308 thinLTOResolvePrevailingInModule(Mod
, DefinedGlobals
);
1314 LLVMRustPrepareThinLTOInternalize(const LLVMRustThinLTOData
*Data
, LLVMModuleRef M
) {
1315 Module
&Mod
= *unwrap(M
);
1316 const auto &DefinedGlobals
= Data
->ModuleToDefinedGVSummaries
.lookup(Mod
.getModuleIdentifier());
1317 thinLTOInternalizeModule(Mod
, DefinedGlobals
);
1322 LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData
*Data
, LLVMModuleRef M
,
1323 LLVMTargetMachineRef TM
) {
1324 Module
&Mod
= *unwrap(M
);
1325 TargetMachine
&Target
= *unwrap(TM
);
1327 const auto &ImportList
= Data
->ImportLists
.lookup(Mod
.getModuleIdentifier());
1328 auto Loader
= [&](StringRef Identifier
) {
1329 const auto &Memory
= Data
->ModuleMap
.lookup(Identifier
);
1330 auto &Context
= Mod
.getContext();
1331 auto MOrErr
= getLazyBitcodeModule(Memory
, Context
, true, true);
1336 // The rest of this closure is a workaround for
1337 // https://bugs.llvm.org/show_bug.cgi?id=38184 where during ThinLTO imports
1338 // we accidentally import wasm custom sections into different modules,
1339 // duplicating them by in the final output artifact.
1341 // The issue is worked around here by manually removing the
1342 // `wasm.custom_sections` named metadata node from any imported module. This
1343 // we know isn't used by any optimization pass so there's no need for it to
1346 // Note that the metadata is currently lazily loaded, so we materialize it
1347 // here before looking up if there's metadata inside. The `FunctionImporter`
1348 // will immediately materialize metadata anyway after an import, so this
1349 // shouldn't be a perf hit.
1350 if (Error Err
= (*MOrErr
)->materializeMetadata()) {
1351 Expected
<std::unique_ptr
<Module
>> Ret(std::move(Err
));
1355 auto *WasmCustomSections
= (*MOrErr
)->getNamedMetadata("wasm.custom_sections");
1356 if (WasmCustomSections
)
1357 WasmCustomSections
->eraseFromParent();
1361 bool ClearDSOLocal
= clearDSOLocalOnDeclarations(Mod
, Target
);
1362 FunctionImporter
Importer(Data
->Index
, Loader
, ClearDSOLocal
);
1363 Expected
<bool> Result
= Importer
.importFunctions(Mod
, ImportList
);
1365 LLVMRustSetLastError(toString(Result
.takeError()).c_str());
1371 // This struct and various functions are sort of a hack right now, but the
1372 // problem is that we've got in-memory LLVM modules after we generate and
1373 // optimize all codegen-units for one compilation in rustc. To be compatible
1374 // with the LTO support above we need to serialize the modules plus their
1375 // ThinLTO summary into memory.
1377 // This structure is basically an owned version of a serialize module, with
1378 // a ThinLTO summary attached.
1379 struct LLVMRustThinLTOBuffer
{
1383 extern "C" LLVMRustThinLTOBuffer
*
1384 LLVMRustThinLTOBufferCreate(LLVMModuleRef M
, bool is_thin
) {
1385 auto Ret
= std::make_unique
<LLVMRustThinLTOBuffer
>();
1387 raw_string_ostream
OS(Ret
->data
);
1391 LoopAnalysisManager LAM
;
1392 FunctionAnalysisManager FAM
;
1393 CGSCCAnalysisManager CGAM
;
1394 ModuleAnalysisManager MAM
;
1395 PB
.registerModuleAnalyses(MAM
);
1396 PB
.registerCGSCCAnalyses(CGAM
);
1397 PB
.registerFunctionAnalyses(FAM
);
1398 PB
.registerLoopAnalyses(LAM
);
1399 PB
.crossRegisterProxies(LAM
, FAM
, CGAM
, MAM
);
1400 ModulePassManager MPM
;
1401 MPM
.addPass(ThinLTOBitcodeWriterPass(OS
, nullptr));
1402 MPM
.run(*unwrap(M
), MAM
);
1404 WriteBitcodeToFile(*unwrap(M
), OS
);
1408 return Ret
.release();
1412 LLVMRustThinLTOBufferFree(LLVMRustThinLTOBuffer
*Buffer
) {
1416 extern "C" const void*
1417 LLVMRustThinLTOBufferPtr(const LLVMRustThinLTOBuffer
*Buffer
) {
1418 return Buffer
->data
.data();
1422 LLVMRustThinLTOBufferLen(const LLVMRustThinLTOBuffer
*Buffer
) {
1423 return Buffer
->data
.length();
1426 // This is what we used to parse upstream bitcode for actual ThinLTO
1427 // processing. We'll call this once per module optimized through ThinLTO, and
1428 // it'll be called concurrently on many threads.
1429 extern "C" LLVMModuleRef
1430 LLVMRustParseBitcodeForLTO(LLVMContextRef Context
,
1433 const char *identifier
) {
1434 StringRef
Data(data
, len
);
1435 MemoryBufferRef
Buffer(Data
, identifier
);
1436 unwrap(Context
)->enableDebugTypeODRUniquing();
1437 Expected
<std::unique_ptr
<Module
>> SrcOrError
=
1438 parseBitcodeFile(Buffer
, *unwrap(Context
));
1440 LLVMRustSetLastError(toString(SrcOrError
.takeError()).c_str());
1443 return wrap(std::move(*SrcOrError
).release());
1446 // Find the bitcode section in the object file data and return it as a slice.
1447 // Fail if the bitcode section is present but empty.
1449 // On success, the return value is the pointer to the start of the slice and
1450 // `out_len` is filled with the (non-zero) length. On failure, the return value
1451 // is `nullptr` and `out_len` is set to zero.
1452 extern "C" const char*
1453 LLVMRustGetBitcodeSliceFromObjectData(const char *data
,
1458 StringRef
Data(data
, len
);
1459 MemoryBufferRef
Buffer(Data
, ""); // The id is unused.
1461 Expected
<MemoryBufferRef
> BitcodeOrError
=
1462 object::IRObjectFile::findBitcodeInMemBuffer(Buffer
);
1463 if (!BitcodeOrError
) {
1464 LLVMRustSetLastError(toString(BitcodeOrError
.takeError()).c_str());
1468 *out_len
= BitcodeOrError
->getBufferSize();
1469 return BitcodeOrError
->getBufferStart();
1472 // Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See
1473 // the comment in `back/lto.rs` for why this exists.
1475 LLVMRustThinLTOGetDICompileUnit(LLVMModuleRef Mod
,
1477 DICompileUnit
**B
) {
1478 Module
*M
= unwrap(Mod
);
1479 DICompileUnit
**Cur
= A
;
1480 DICompileUnit
**Next
= B
;
1481 for (DICompileUnit
*CU
: M
->debug_compile_units()) {
1490 // Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See
1491 // the comment in `back/lto.rs` for why this exists.
1493 LLVMRustThinLTOPatchDICompileUnit(LLVMModuleRef Mod
, DICompileUnit
*Unit
) {
1494 Module
*M
= unwrap(Mod
);
1496 // If the original source module didn't have a `DICompileUnit` then try to
1497 // merge all the existing compile units. If there aren't actually any though
1498 // then there's not much for us to do so return.
1499 if (Unit
== nullptr) {
1500 for (DICompileUnit
*CU
: M
->debug_compile_units()) {
1504 if (Unit
== nullptr)
1508 // Use LLVM's built-in `DebugInfoFinder` to find a bunch of debuginfo and
1509 // process it recursively. Note that we used to specifically iterate over
1510 // instructions to ensure we feed everything into it, but `processModule`
1511 // started doing this the same way in LLVM 7 (commit d769eb36ab2b8).
1512 DebugInfoFinder Finder
;
1513 Finder
.processModule(*M
);
1515 // After we've found all our debuginfo, rewrite all subprograms to point to
1516 // the same `DICompileUnit`.
1517 for (auto &F
: Finder
.subprograms()) {
1518 F
->replaceUnit(Unit
);
1521 // Erase any other references to other `DICompileUnit` instances, the verifier
1522 // will later ensure that we don't actually have any other stale references to
1524 auto *MD
= M
->getNamedMetadata("llvm.dbg.cu");
1525 MD
->clearOperands();
1526 MD
->addOperand(Unit
);
1529 // Computes the LTO cache key for the provided 'ModId' in the given 'Data',
1530 // storing the result in 'KeyOut'.
1531 // Currently, this cache key is a SHA-1 hash of anything that could affect
1532 // the result of optimizing this module (e.g. module imports, exports, liveness
1533 // of access globals, etc).
1534 // The precise details are determined by LLVM in `computeLTOCacheKey`, which is
1535 // used during the normal linker-plugin incremental thin-LTO process.
1537 LLVMRustComputeLTOCacheKey(RustStringRef KeyOut
, const char *ModId
, LLVMRustThinLTOData
*Data
) {
1538 SmallString
<40> Key
;
1539 llvm::lto::Config conf
;
1540 const auto &ImportList
= Data
->ImportLists
.lookup(ModId
);
1541 const auto &ExportList
= Data
->ExportLists
.lookup(ModId
);
1542 const auto &ResolvedODR
= Data
->ResolvedODR
.lookup(ModId
);
1543 const auto &DefinedGlobals
= Data
->ModuleToDefinedGVSummaries
.lookup(ModId
);
1544 std::set
<GlobalValue::GUID
> CfiFunctionDefs
;
1545 std::set
<GlobalValue::GUID
> CfiFunctionDecls
;
1547 // Based on the 'InProcessThinBackend' constructor in LLVM
1548 for (auto &Name
: Data
->Index
.cfiFunctionDefs())
1549 CfiFunctionDefs
.insert(
1550 GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name
)));
1551 for (auto &Name
: Data
->Index
.cfiFunctionDecls())
1552 CfiFunctionDecls
.insert(
1553 GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name
)));
1555 llvm::computeLTOCacheKey(Key
, conf
, Data
->Index
, ModId
,
1556 ImportList
, ExportList
, ResolvedODR
, DefinedGlobals
, CfiFunctionDefs
, CfiFunctionDecls
1559 LLVMRustStringWriteImpl(KeyOut
, Key
.c_str(), Key
.size());