]>
Commit | Line | Data |
---|---|---|
9fa01778 | 1 | use crate::back::write::create_informational_target_machine; |
cdc7bbd5 | 2 | use crate::{llvm, llvm_util}; |
dfeec247 | 3 | use libc::c_int; |
29967ef6 | 4 | use rustc_codegen_ssa::target_features::supported_target_features; |
dfeec247 | 5 | use rustc_data_structures::fx::FxHashSet; |
136023e0 | 6 | use rustc_metadata::dynamic_lib::DynamicLibrary; |
ba9703b0 XL |
7 | use rustc_middle::bug; |
8 | use rustc_session::config::PrintRequest; | |
9 | use rustc_session::Session; | |
dfeec247 | 10 | use rustc_span::symbol::Symbol; |
e74abb32 | 11 | use rustc_target::spec::{MergeFunctions, PanicStrategy}; |
5869c6ff | 12 | use std::ffi::{CStr, CString}; |
136023e0 | 13 | use tracing::debug; |
7cac9316 | 14 | |
136023e0 XL |
15 | use std::mem; |
16 | use std::path::Path; | |
cdc7bbd5 | 17 | use std::ptr; |
b7449926 | 18 | use std::slice; |
dfeec247 | 19 | use std::str; |
7cac9316 XL |
20 | use std::sync::Once; |
21 | ||
2c00a5a8 XL |
22 | static INIT: Once = Once::new(); |
23 | ||
24 | pub(crate) fn init(sess: &Session) { | |
7cac9316 XL |
25 | unsafe { |
26 | // Before we touch LLVM, make sure that multithreading is enabled. | |
3c0e092e XL |
27 | if llvm::LLVMIsMultithreaded() != 1 { |
28 | bug!("LLVM compiled without support for threads"); | |
29 | } | |
7cac9316 | 30 | INIT.call_once(|| { |
7cac9316 XL |
31 | configure_llvm(sess); |
32 | }); | |
7cac9316 XL |
33 | } |
34 | } | |
35 | ||
2c00a5a8 | 36 | fn require_inited() { |
3c0e092e XL |
37 | if !INIT.is_completed() { |
38 | bug!("LLVM is not initialized"); | |
2c00a5a8 XL |
39 | } |
40 | } | |
41 | ||
7cac9316 | 42 | unsafe fn configure_llvm(sess: &Session) { |
29967ef6 | 43 | let n_args = sess.opts.cg.llvm_args.len() + sess.target.llvm_args.len(); |
0bf4aa26 XL |
44 | let mut llvm_c_strs = Vec::with_capacity(n_args + 1); |
45 | let mut llvm_args = Vec::with_capacity(n_args + 1); | |
46 | ||
47 | llvm::LLVMRustInstallFatalErrorHandler(); | |
7cac9316 | 48 | |
dfeec247 XL |
49 | fn llvm_arg_to_arg_name(full_arg: &str) -> &str { |
50 | full_arg.trim().split(|c: char| c == '=' || c.is_whitespace()).next().unwrap_or("") | |
51 | } | |
52 | ||
53 | let cg_opts = sess.opts.cg.llvm_args.iter(); | |
29967ef6 | 54 | let tg_opts = sess.target.llvm_args.iter(); |
dfeec247 XL |
55 | let sess_args = cg_opts.chain(tg_opts); |
56 | ||
57 | let user_specified_args: FxHashSet<_> = | |
74b04a01 | 58 | sess_args.clone().map(|s| llvm_arg_to_arg_name(s)).filter(|s| !s.is_empty()).collect(); |
dfeec247 | 59 | |
7cac9316 | 60 | { |
dfeec247 XL |
61 | // This adds the given argument to LLVM. Unless `force` is true |
62 | // user specified arguments are *not* overridden. | |
63 | let mut add = |arg: &str, force: bool| { | |
64 | if force || !user_specified_args.contains(llvm_arg_to_arg_name(arg)) { | |
65 | let s = CString::new(arg).unwrap(); | |
66 | llvm_args.push(s.as_ptr()); | |
67 | llvm_c_strs.push(s); | |
68 | } | |
7cac9316 | 69 | }; |
3dfed10e XL |
70 | // Set the llvm "program name" to make usage and invalid argument messages more clear. |
71 | add("rustc -Cllvm-args=\"...\" with", true); | |
dfeec247 XL |
72 | if sess.time_llvm_passes() { |
73 | add("-time-passes", false); | |
0531ce1d | 74 | } |
dfeec247 XL |
75 | if sess.print_llvm_passes() { |
76 | add("-debug-pass=Structure", false); | |
77 | } | |
3c0e092e XL |
78 | if sess.target.generate_arange_section |
79 | && !sess.opts.debugging_opts.no_generate_arange_section | |
80 | { | |
dfeec247 | 81 | add("-generate-arange-section", false); |
60c5eb7d | 82 | } |
cdc7bbd5 | 83 | |
94222f64 XL |
84 | // Disable the machine outliner by default in LLVM versions 11 and LLVM |
85 | // version 12, where it leads to miscompilation. | |
cdc7bbd5 | 86 | // |
94222f64 XL |
87 | // Ref: |
88 | // - https://github.com/rust-lang/rust/issues/85351 | |
89 | // - https://reviews.llvm.org/D103167 | |
3c0e092e | 90 | if llvm_util::get_version() < (13, 0, 0) { |
cdc7bbd5 XL |
91 | add("-enable-machine-outliner=never", false); |
92 | } | |
93 | ||
29967ef6 | 94 | match sess.opts.debugging_opts.merge_functions.unwrap_or(sess.target.merge_functions) { |
ba9703b0 XL |
95 | MergeFunctions::Disabled | MergeFunctions::Trampolines => {} |
96 | MergeFunctions::Aliases => { | |
97 | add("-mergefunc-use-aliases", false); | |
9fa01778 | 98 | } |
a1dfa0c6 | 99 | } |
7cac9316 | 100 | |
29967ef6 | 101 | if sess.target.os == "emscripten" && sess.panic_strategy() == PanicStrategy::Unwind { |
dfeec247 | 102 | add("-enable-emscripten-cxx-exceptions", false); |
e74abb32 XL |
103 | } |
104 | ||
0731742a XL |
105 | // HACK(eddyb) LLVM inserts `llvm.assume` calls to preserve align attributes |
106 | // during inlining. Unfortunately these may block other optimizations. | |
dfeec247 | 107 | add("-preserve-alignment-assumptions-during-inlining=false", false); |
0731742a | 108 | |
cdc7bbd5 XL |
109 | // Use non-zero `import-instr-limit` multiplier for cold callsites. |
110 | add("-import-cold-multiplier=0.1", false); | |
111 | ||
dfeec247 XL |
112 | for arg in sess_args { |
113 | add(&(*arg), true); | |
7cac9316 XL |
114 | } |
115 | } | |
116 | ||
fc512014 | 117 | if sess.opts.debugging_opts.llvm_time_trace { |
74b04a01 XL |
118 | llvm::LLVMTimeTraceProfilerInitialize(); |
119 | } | |
120 | ||
7cac9316 XL |
121 | llvm::LLVMInitializePasses(); |
122 | ||
136023e0 XL |
123 | for plugin in &sess.opts.debugging_opts.llvm_plugins { |
124 | let path = Path::new(plugin); | |
125 | let res = DynamicLibrary::open(path); | |
126 | match res { | |
127 | Ok(_) => debug!("LLVM plugin loaded succesfully {} ({})", path.display(), plugin), | |
128 | Err(e) => bug!("couldn't load plugin: {}", e), | |
129 | } | |
130 | mem::forget(res); | |
131 | } | |
132 | ||
29967ef6 | 133 | rustc_llvm::initialize_available_targets(); |
7cac9316 | 134 | |
dfeec247 | 135 | llvm::LLVMRustSetLLVMOptions(llvm_args.len() as c_int, llvm_args.as_ptr()); |
7cac9316 XL |
136 | } |
137 | ||
74b04a01 XL |
138 | pub fn time_trace_profiler_finish(file_name: &str) { |
139 | unsafe { | |
fc512014 XL |
140 | let file_name = CString::new(file_name).unwrap(); |
141 | llvm::LLVMTimeTraceProfilerFinish(file_name.as_ptr()); | |
74b04a01 XL |
142 | } |
143 | } | |
144 | ||
0531ce1d XL |
145 | // WARNING: the features after applying `to_llvm_feature` must be known |
146 | // to LLVM or the feature detection code will walk past the end of the feature | |
147 | // array, leading to crashes. | |
fc512014 XL |
148 | // To find a list of LLVM's names, check llvm-project/llvm/include/llvm/Support/*TargetParser.def |
149 | // where the * matches the architecture's name | |
150 | // Beware to not use the llvm github project for this, but check the git submodule | |
151 | // found in src/llvm-project | |
152 | // Though note that Rust can also be build with an external precompiled version of LLVM | |
153 | // which might lead to failures if the oldest tested / supported LLVM version | |
154 | // doesn't yet support the relevant intrinsics | |
c295e0f8 | 155 | pub fn to_llvm_feature<'a>(sess: &Session, s: &'a str) -> Vec<&'a str> { |
29967ef6 | 156 | let arch = if sess.target.arch == "x86_64" { "x86" } else { &*sess.target.arch }; |
0531ce1d | 157 | match (arch, s) { |
c295e0f8 XL |
158 | ("x86", "sse4.2") => { |
159 | if get_version() >= (14, 0, 0) { | |
160 | vec!["sse4.2", "crc32"] | |
161 | } else { | |
162 | vec!["sse4.2"] | |
163 | } | |
164 | } | |
165 | ("x86", "pclmulqdq") => vec!["pclmul"], | |
166 | ("x86", "rdrand") => vec!["rdrnd"], | |
167 | ("x86", "bmi1") => vec!["bmi"], | |
168 | ("x86", "cmpxchg16b") => vec!["cx16"], | |
169 | ("x86", "avx512vaes") => vec!["vaes"], | |
170 | ("x86", "avx512gfni") => vec!["gfni"], | |
171 | ("x86", "avx512vpclmulqdq") => vec!["vpclmulqdq"], | |
172 | ("aarch64", "fp") => vec!["fp-armv8"], | |
173 | ("aarch64", "fp16") => vec!["fullfp16"], | |
174 | ("aarch64", "fhm") => vec!["fp16fml"], | |
175 | ("aarch64", "rcpc2") => vec!["rcpc-immo"], | |
176 | ("aarch64", "dpb") => vec!["ccpp"], | |
177 | ("aarch64", "dpb2") => vec!["ccdp"], | |
178 | ("aarch64", "frintts") => vec!["fptoint"], | |
179 | ("aarch64", "fcma") => vec!["complxnum"], | |
3c0e092e | 180 | ("aarch64", "pmuv3") => vec!["perfmon"], |
c295e0f8 | 181 | (_, s) => vec![s], |
0531ce1d XL |
182 | } |
183 | } | |
ff7c6d11 | 184 | |
7cac9316 | 185 | pub fn target_features(sess: &Session) -> Vec<Symbol> { |
f9f354fc | 186 | let target_machine = create_informational_target_machine(sess); |
f035d41b | 187 | supported_target_features(sess) |
0531ce1d | 188 | .iter() |
fc512014 XL |
189 | .filter_map( |
190 | |&(feature, gate)| { | |
191 | if sess.is_nightly_build() || gate.is_none() { Some(feature) } else { None } | |
192 | }, | |
193 | ) | |
0531ce1d | 194 | .filter(|feature| { |
c295e0f8 XL |
195 | for llvm_feature in to_llvm_feature(sess, feature) { |
196 | let cstr = CString::new(llvm_feature).unwrap(); | |
197 | if unsafe { llvm::LLVMRustHasFeature(target_machine, cstr.as_ptr()) } { | |
198 | return true; | |
199 | } | |
200 | } | |
201 | false | |
0531ce1d | 202 | }) |
dfeec247 XL |
203 | .map(|feature| Symbol::intern(feature)) |
204 | .collect() | |
2c00a5a8 | 205 | } |
7cac9316 | 206 | |
7cac9316 | 207 | pub fn print_version() { |
5869c6ff XL |
208 | let (major, minor, patch) = get_version(); |
209 | println!("LLVM version: {}.{}.{}", major, minor, patch); | |
210 | } | |
211 | ||
212 | pub fn get_version() -> (u32, u32, u32) { | |
2c00a5a8 | 213 | // Can be called without initializing LLVM |
7cac9316 | 214 | unsafe { |
5869c6ff | 215 | (llvm::LLVMRustVersionMajor(), llvm::LLVMRustVersionMinor(), llvm::LLVMRustVersionPatch()) |
7cac9316 XL |
216 | } |
217 | } | |
218 | ||
219 | pub fn print_passes() { | |
2c00a5a8 | 220 | // Can be called without initializing LLVM |
dfeec247 XL |
221 | unsafe { |
222 | llvm::LLVMRustPrintPasses(); | |
223 | } | |
7cac9316 XL |
224 | } |
225 | ||
cdc7bbd5 XL |
226 | fn llvm_target_features(tm: &llvm::TargetMachine) -> Vec<(&str, &str)> { |
227 | let len = unsafe { llvm::LLVMRustGetTargetFeaturesCount(tm) }; | |
228 | let mut ret = Vec::with_capacity(len); | |
229 | for i in 0..len { | |
230 | unsafe { | |
231 | let mut feature = ptr::null(); | |
232 | let mut desc = ptr::null(); | |
233 | llvm::LLVMRustGetTargetFeature(tm, i, &mut feature, &mut desc); | |
234 | if feature.is_null() || desc.is_null() { | |
235 | bug!("LLVM returned a `null` target feature string"); | |
236 | } | |
237 | let feature = CStr::from_ptr(feature).to_str().unwrap_or_else(|e| { | |
238 | bug!("LLVM returned a non-utf8 feature string: {}", e); | |
239 | }); | |
240 | let desc = CStr::from_ptr(desc).to_str().unwrap_or_else(|e| { | |
241 | bug!("LLVM returned a non-utf8 feature string: {}", e); | |
242 | }); | |
243 | ret.push((feature, desc)); | |
244 | } | |
245 | } | |
246 | ret | |
247 | } | |
248 | ||
249 | fn print_target_features(sess: &Session, tm: &llvm::TargetMachine) { | |
250 | let mut target_features = llvm_target_features(tm); | |
251 | let mut rustc_target_features = supported_target_features(sess) | |
252 | .iter() | |
253 | .filter_map(|(feature, _gate)| { | |
c295e0f8 XL |
254 | for llvm_feature in to_llvm_feature(sess, *feature) { |
255 | // LLVM asserts that these are sorted. LLVM and Rust both use byte comparison for these strings. | |
256 | match target_features.binary_search_by_key(&llvm_feature, |(f, _d)| (*f)).ok().map( | |
257 | |index| { | |
258 | let (_f, desc) = target_features.remove(index); | |
259 | (*feature, desc) | |
260 | }, | |
261 | ) { | |
262 | Some(v) => return Some(v), | |
263 | None => {} | |
264 | } | |
265 | } | |
266 | None | |
cdc7bbd5 XL |
267 | }) |
268 | .collect::<Vec<_>>(); | |
269 | rustc_target_features.extend_from_slice(&[( | |
270 | "crt-static", | |
271 | "Enables C Run-time Libraries to be statically linked", | |
272 | )]); | |
273 | let max_feature_len = target_features | |
274 | .iter() | |
275 | .chain(rustc_target_features.iter()) | |
276 | .map(|(feature, _desc)| feature.len()) | |
277 | .max() | |
278 | .unwrap_or(0); | |
279 | ||
280 | println!("Features supported by rustc for this target:"); | |
281 | for (feature, desc) in &rustc_target_features { | |
282 | println!(" {1:0$} - {2}.", max_feature_len, feature, desc); | |
283 | } | |
284 | println!("\nCode-generation features supported by LLVM for this target:"); | |
285 | for (feature, desc) in &target_features { | |
286 | println!(" {1:0$} - {2}.", max_feature_len, feature, desc); | |
287 | } | |
c295e0f8 | 288 | if target_features.is_empty() { |
cdc7bbd5 XL |
289 | println!(" Target features listing is not supported by this LLVM version."); |
290 | } | |
291 | println!("\nUse +feature to enable a feature, or -feature to disable it."); | |
292 | println!("For example, rustc -C target-cpu=mycpu -C target-feature=+feature1,-feature2\n"); | |
293 | println!("Code-generation features cannot be used in cfg or #[target_feature],"); | |
294 | println!("and may be renamed or removed in a future version of LLVM or rustc.\n"); | |
295 | } | |
296 | ||
2c00a5a8 XL |
297 | pub(crate) fn print(req: PrintRequest, sess: &Session) { |
298 | require_inited(); | |
f9f354fc | 299 | let tm = create_informational_target_machine(sess); |
cdc7bbd5 XL |
300 | match req { |
301 | PrintRequest::TargetCPUs => unsafe { llvm::LLVMRustPrintTargetCPUs(tm) }, | |
302 | PrintRequest::TargetFeatures => print_target_features(sess, tm), | |
303 | _ => bug!("rustc_codegen_llvm can't handle print request: {:?}", req), | |
7cac9316 XL |
304 | } |
305 | } | |
b7449926 | 306 | |
29967ef6 | 307 | fn handle_native(name: &str) -> &str { |
b7449926 | 308 | if name != "native" { |
dfeec247 | 309 | return name; |
b7449926 XL |
310 | } |
311 | ||
312 | unsafe { | |
313 | let mut len = 0; | |
314 | let ptr = llvm::LLVMRustGetHostCPUName(&mut len); | |
315 | str::from_utf8(slice::from_raw_parts(ptr as *const u8, len)).unwrap() | |
316 | } | |
317 | } | |
29967ef6 XL |
318 | |
319 | pub fn target_cpu(sess: &Session) -> &str { | |
5869c6ff | 320 | let name = sess.opts.cg.target_cpu.as_ref().unwrap_or(&sess.target.cpu); |
29967ef6 XL |
321 | handle_native(name) |
322 | } | |
323 | ||
6a06907d XL |
324 | /// The list of LLVM features computed from CLI flags (`-Ctarget-cpu`, `-Ctarget-feature`, |
325 | /// `--target` and similar). | |
326 | // FIXME(nagisa): Cache the output of this somehow? Maybe make this a query? We're calling this | |
327 | // for every function that has `#[target_feature]` on it. The global features won't change between | |
328 | // the functions; only crates, maybe… | |
329 | pub fn llvm_global_features(sess: &Session) -> Vec<String> { | |
330 | // FIXME(nagisa): this should definitely be available more centrally and to other codegen backends. | |
331 | /// These features control behaviour of rustc rather than llvm. | |
332 | const RUSTC_SPECIFIC_FEATURES: &[&str] = &["crt-static"]; | |
333 | ||
334 | // Features that come earlier are overriden by conflicting features later in the string. | |
335 | // Typically we'll want more explicit settings to override the implicit ones, so: | |
336 | // | |
337 | // * Features from -Ctarget-cpu=*; are overriden by [^1] | |
338 | // * Features implied by --target; are overriden by | |
339 | // * Features from -Ctarget-feature; are overriden by | |
340 | // * function specific features. | |
341 | // | |
342 | // [^1]: target-cpu=native is handled here, other target-cpu values are handled implicitly | |
343 | // through LLVM TargetMachine implementation. | |
344 | // | |
345 | // FIXME(nagisa): it isn't clear what's the best interaction between features implied by | |
346 | // `-Ctarget-cpu` and `--target` are. On one hand, you'd expect CLI arguments to always | |
347 | // override anything that's implicit, so e.g. when there's no `--target` flag, features implied | |
348 | // the host target are overriden by `-Ctarget-cpu=*`. On the other hand, what about when both | |
349 | // `--target` and `-Ctarget-cpu=*` are specified? Both then imply some target features and both | |
350 | // flags are specified by the user on the CLI. It isn't as clear-cut which order of precedence | |
351 | // should be taken in cases like these. | |
352 | let mut features = vec![]; | |
353 | ||
354 | // -Ctarget-cpu=native | |
5869c6ff | 355 | match sess.opts.cg.target_cpu { |
6a06907d | 356 | Some(ref s) if s == "native" => { |
5869c6ff XL |
357 | let features_string = unsafe { |
358 | let ptr = llvm::LLVMGetHostCPUFeatures(); | |
359 | let features_string = if !ptr.is_null() { | |
360 | CStr::from_ptr(ptr) | |
361 | .to_str() | |
362 | .unwrap_or_else(|e| { | |
363 | bug!("LLVM returned a non-utf8 features string: {}", e); | |
364 | }) | |
365 | .to_owned() | |
366 | } else { | |
367 | bug!("could not allocate host CPU features, LLVM returned a `null` string"); | |
368 | }; | |
369 | ||
370 | llvm::LLVMDisposeMessage(ptr); | |
371 | ||
372 | features_string | |
373 | }; | |
94222f64 | 374 | features.extend(features_string.split(',').map(String::from)); |
5869c6ff | 375 | } |
6a06907d XL |
376 | Some(_) | None => {} |
377 | }; | |
378 | ||
17df50a5 XL |
379 | let filter = |s: &str| { |
380 | if s.is_empty() { | |
c295e0f8 | 381 | return vec![]; |
17df50a5 | 382 | } |
94222f64 | 383 | let feature = if s.starts_with('+') || s.starts_with('-') { |
17df50a5 XL |
384 | &s[1..] |
385 | } else { | |
c295e0f8 | 386 | return vec![s.to_string()]; |
17df50a5 XL |
387 | }; |
388 | // Rustc-specific feature requests like `+crt-static` or `-crt-static` | |
389 | // are not passed down to LLVM. | |
390 | if RUSTC_SPECIFIC_FEATURES.contains(&feature) { | |
c295e0f8 | 391 | return vec![]; |
17df50a5 XL |
392 | } |
393 | // ... otherwise though we run through `to_llvm_feature` feature when | |
394 | // passing requests down to LLVM. This means that all in-language | |
395 | // features also work on the command line instead of having two | |
396 | // different names when the LLVM name and the Rust name differ. | |
c295e0f8 | 397 | to_llvm_feature(sess, feature).iter().map(|f| format!("{}{}", &s[..1], f)).collect() |
17df50a5 XL |
398 | }; |
399 | ||
6a06907d | 400 | // Features implied by an implicit or explicit `--target`. |
c295e0f8 | 401 | features.extend(sess.target.features.split(',').flat_map(&filter)); |
6a06907d XL |
402 | |
403 | // -Ctarget-features | |
c295e0f8 XL |
404 | features.extend(sess.opts.cg.target_feature.split(',').flat_map(&filter)); |
405 | ||
6a06907d | 406 | features |
29967ef6 | 407 | } |
5869c6ff XL |
408 | |
409 | pub fn tune_cpu(sess: &Session) -> Option<&str> { | |
410 | let name = sess.opts.debugging_opts.tune_cpu.as_ref()?; | |
411 | Some(handle_native(name)) | |
412 | } |