]>
Commit | Line | Data |
---|---|---|
5e7ed085 | 1 | use crate::attributes; |
9fa01778 XL |
2 | use crate::builder::Builder; |
3 | use crate::context::CodegenCx; | |
5e7ed085 | 4 | use crate::llvm::{self, Attribute, AttributePlace}; |
9fa01778 | 5 | use crate::type_::Type; |
60c5eb7d | 6 | use crate::type_of::LayoutLlvmExt; |
9fa01778 | 7 | use crate::value::Value; |
60c5eb7d | 8 | |
dfeec247 XL |
9 | use rustc_codegen_ssa::mir::operand::OperandValue; |
10 | use rustc_codegen_ssa::mir::place::PlaceRef; | |
a1dfa0c6 | 11 | use rustc_codegen_ssa::traits::*; |
dfeec247 | 12 | use rustc_codegen_ssa::MemFlags; |
ba9703b0 | 13 | use rustc_middle::bug; |
c295e0f8 | 14 | use rustc_middle::ty::layout::LayoutOf; |
ba9703b0 XL |
15 | pub use rustc_middle::ty::layout::{FAT_PTR_ADDR, FAT_PTR_EXTRA}; |
16 | use rustc_middle::ty::Ty; | |
5e7ed085 | 17 | use rustc_session::config; |
60c5eb7d | 18 | use rustc_target::abi::call::ArgAbi; |
83c7162d | 19 | pub use rustc_target::abi::call::*; |
c295e0f8 | 20 | use rustc_target::abi::{self, HasDataLayout, Int}; |
dfeec247 | 21 | pub use rustc_target::spec::abi::Abi; |
476ff2be | 22 | |
ba9703b0 | 23 | use libc::c_uint; |
5e7ed085 | 24 | use smallvec::SmallVec; |
476ff2be | 25 | |
83c7162d | 26 | pub trait ArgAttributesExt { |
cdc7bbd5 XL |
27 | fn apply_attrs_to_llfn(&self, idx: AttributePlace, cx: &CodegenCx<'_, '_>, llfn: &Value); |
28 | fn apply_attrs_to_callsite( | |
29 | &self, | |
30 | idx: AttributePlace, | |
31 | cx: &CodegenCx<'_, '_>, | |
32 | callsite: &Value, | |
33 | ); | |
34 | } | |
35 | ||
36 | fn should_use_mutable_noalias(cx: &CodegenCx<'_, '_>) -> bool { | |
3c0e092e XL |
37 | // LLVM prior to version 12 had known miscompiles in the presence of |
38 | // noalias attributes (see #54878), but we don't support earlier | |
39 | // versions at all anymore. We now enable mutable noalias by default. | |
064997fb | 40 | cx.tcx.sess.opts.unstable_opts.mutable_noalias.unwrap_or(true) |
476ff2be SL |
41 | } |
42 | ||
5e7ed085 FG |
43 | const ABI_AFFECTING_ATTRIBUTES: [(ArgAttribute, llvm::AttributeKind); 1] = |
44 | [(ArgAttribute::InReg, llvm::AttributeKind::InReg)]; | |
45 | ||
46 | const OPTIMIZATION_ATTRIBUTES: [(ArgAttribute, llvm::AttributeKind); 5] = [ | |
47 | (ArgAttribute::NoAlias, llvm::AttributeKind::NoAlias), | |
48 | (ArgAttribute::NoCapture, llvm::AttributeKind::NoCapture), | |
49 | (ArgAttribute::NonNull, llvm::AttributeKind::NonNull), | |
50 | (ArgAttribute::ReadOnly, llvm::AttributeKind::ReadOnly), | |
51 | (ArgAttribute::NoUndef, llvm::AttributeKind::NoUndef), | |
52 | ]; | |
53 | ||
54 | fn get_attrs<'ll>(this: &ArgAttributes, cx: &CodegenCx<'ll, '_>) -> SmallVec<[&'ll Attribute; 8]> { | |
55 | let mut regular = this.regular; | |
56 | ||
57 | let mut attrs = SmallVec::new(); | |
58 | ||
59 | // ABI-affecting attributes must always be applied | |
60 | for (attr, llattr) in ABI_AFFECTING_ATTRIBUTES { | |
61 | if regular.contains(attr) { | |
62 | attrs.push(llattr.create_attr(cx.llcx)); | |
63 | } | |
64 | } | |
65 | if let Some(align) = this.pointee_align { | |
66 | attrs.push(llvm::CreateAlignmentAttr(cx.llcx, align.bytes())); | |
67 | } | |
68 | match this.arg_ext { | |
69 | ArgExtension::None => {} | |
70 | ArgExtension::Zext => attrs.push(llvm::AttributeKind::ZExt.create_attr(cx.llcx)), | |
71 | ArgExtension::Sext => attrs.push(llvm::AttributeKind::SExt.create_attr(cx.llcx)), | |
72 | } | |
73 | ||
74 | // Only apply remaining attributes when optimizing | |
75 | if cx.sess().opts.optimize != config::OptLevel::No { | |
76 | let deref = this.pointee_size.bytes(); | |
77 | if deref != 0 { | |
78 | if regular.contains(ArgAttribute::NonNull) { | |
79 | attrs.push(llvm::CreateDereferenceableAttr(cx.llcx, deref)); | |
80 | } else { | |
81 | attrs.push(llvm::CreateDereferenceableOrNullAttr(cx.llcx, deref)); | |
cdc7bbd5 | 82 | } |
5e7ed085 FG |
83 | regular -= ArgAttribute::NonNull; |
84 | } | |
85 | for (attr, llattr) in OPTIMIZATION_ATTRIBUTES { | |
86 | if regular.contains(attr) { | |
87 | attrs.push(llattr.create_attr(cx.llcx)); | |
fc512014 | 88 | } |
476ff2be | 89 | } |
5e7ed085 FG |
90 | if regular.contains(ArgAttribute::NoAliasMutRef) && should_use_mutable_noalias(cx) { |
91 | attrs.push(llvm::AttributeKind::NoAlias.create_attr(cx.llcx)); | |
92 | } | |
93 | } | |
94 | ||
95 | attrs | |
96 | } | |
97 | ||
98 | impl ArgAttributesExt for ArgAttributes { | |
99 | fn apply_attrs_to_llfn(&self, idx: AttributePlace, cx: &CodegenCx<'_, '_>, llfn: &Value) { | |
100 | let attrs = get_attrs(self, cx); | |
101 | attributes::apply_to_llfn(llfn, idx, &attrs); | |
476ff2be SL |
102 | } |
103 | ||
cdc7bbd5 XL |
104 | fn apply_attrs_to_callsite( |
105 | &self, | |
106 | idx: AttributePlace, | |
107 | cx: &CodegenCx<'_, '_>, | |
108 | callsite: &Value, | |
109 | ) { | |
5e7ed085 FG |
110 | let attrs = get_attrs(self, cx); |
111 | attributes::apply_to_callsite(callsite, idx, &attrs); | |
476ff2be SL |
112 | } |
113 | } | |
cc61c64b | 114 | |
83c7162d | 115 | pub trait LlvmType { |
a2a8927a | 116 | fn llvm_type<'ll>(&self, cx: &CodegenCx<'ll, '_>) -> &'ll Type; |
cc61c64b XL |
117 | } |
118 | ||
83c7162d | 119 | impl LlvmType for Reg { |
a2a8927a | 120 | fn llvm_type<'ll>(&self, cx: &CodegenCx<'ll, '_>) -> &'ll Type { |
cc61c64b | 121 | match self.kind { |
a1dfa0c6 | 122 | RegKind::Integer => cx.type_ix(self.size.bits()), |
dfeec247 XL |
123 | RegKind::Float => match self.size.bits() { |
124 | 32 => cx.type_f32(), | |
125 | 64 => cx.type_f64(), | |
126 | _ => bug!("unsupported float: {:?}", self), | |
127 | }, | |
128 | RegKind::Vector => cx.type_vector(cx.type_i8(), self.size.bytes()), | |
cc61c64b XL |
129 | } |
130 | } | |
131 | } | |
132 | ||
83c7162d | 133 | impl LlvmType for CastTarget { |
a2a8927a | 134 | fn llvm_type<'ll>(&self, cx: &CodegenCx<'ll, '_>) -> &'ll Type { |
0531ce1d | 135 | let rest_ll_unit = self.rest.unit.llvm_type(cx); |
94b46f34 XL |
136 | let (rest_count, rem_bytes) = if self.rest.unit.size.bytes() == 0 { |
137 | (0, 0) | |
138 | } else { | |
dfeec247 XL |
139 | ( |
140 | self.rest.total.bytes() / self.rest.unit.size.bytes(), | |
141 | self.rest.total.bytes() % self.rest.unit.size.bytes(), | |
142 | ) | |
94b46f34 | 143 | }; |
0531ce1d XL |
144 | |
145 | if self.prefix.iter().all(|x| x.is_none()) { | |
146 | // Simplify to a single unit when there is no prefix and size <= unit size | |
147 | if self.rest.total <= self.rest.unit.size { | |
148 | return rest_ll_unit; | |
149 | } | |
150 | ||
151 | // Simplify to array when all chunks are the same size and type | |
152 | if rem_bytes == 0 { | |
a1dfa0c6 | 153 | return cx.type_array(rest_ll_unit, rest_count); |
cc61c64b XL |
154 | } |
155 | } | |
0531ce1d XL |
156 | |
157 | // Create list of fields in the main structure | |
dfeec247 XL |
158 | let mut args: Vec<_> = self |
159 | .prefix | |
160 | .iter() | |
a2a8927a | 161 | .flat_map(|option_reg| option_reg.map(|reg| reg.llvm_type(cx))) |
0531ce1d XL |
162 | .chain((0..rest_count).map(|_| rest_ll_unit)) |
163 | .collect(); | |
164 | ||
165 | // Append final integer | |
166 | if rem_bytes != 0 { | |
167 | // Only integers can be really split further. | |
168 | assert_eq!(self.rest.unit.kind, RegKind::Integer); | |
a1dfa0c6 | 169 | args.push(cx.type_ix(rem_bytes * 8)); |
0531ce1d XL |
170 | } |
171 | ||
a1dfa0c6 | 172 | cx.type_struct(&args, false) |
cc61c64b XL |
173 | } |
174 | } | |
476ff2be | 175 | |
60c5eb7d | 176 | pub trait ArgAbiExt<'ll, 'tcx> { |
b7449926 | 177 | fn memory_ty(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type; |
a1dfa0c6 XL |
178 | fn store( |
179 | &self, | |
180 | bx: &mut Builder<'_, 'll, 'tcx>, | |
181 | val: &'ll Value, | |
182 | dst: PlaceRef<'tcx, &'ll Value>, | |
183 | ); | |
184 | fn store_fn_arg( | |
185 | &self, | |
186 | bx: &mut Builder<'_, 'll, 'tcx>, | |
187 | idx: &mut usize, | |
188 | dst: PlaceRef<'tcx, &'ll Value>, | |
189 | ); | |
54a0048b SL |
190 | } |
191 | ||
a2a8927a | 192 | impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> { |
9fa01778 | 193 | /// Gets the LLVM type for a place of the original Rust type of |
0731742a | 194 | /// this argument/return, i.e., the result of `type_of::type_of`. |
b7449926 | 195 | fn memory_ty(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type { |
2c00a5a8 | 196 | self.layout.llvm_type(cx) |
54a0048b SL |
197 | } |
198 | ||
60c5eb7d | 199 | /// Stores a direct/indirect value described by this ArgAbi into a |
ff7c6d11 | 200 | /// place for the original Rust type of this argument/return. |
54a0048b SL |
201 | /// Can be used for both storing formal arguments into Rust variables |
202 | /// or results of call/invoke instructions into their destinations. | |
a1dfa0c6 XL |
203 | fn store( |
204 | &self, | |
205 | bx: &mut Builder<'_, 'll, 'tcx>, | |
206 | val: &'ll Value, | |
207 | dst: PlaceRef<'tcx, &'ll Value>, | |
208 | ) { | |
54a0048b SL |
209 | if self.is_ignore() { |
210 | return; | |
211 | } | |
b7449926 | 212 | if self.is_sized_indirect() { |
a1dfa0c6 | 213 | OperandValue::Ref(val, None, self.layout.align.abi).store(bx, dst) |
b7449926 | 214 | } else if self.is_unsized_indirect() { |
60c5eb7d | 215 | bug!("unsized `ArgAbi` must be handled through `store_fn_arg`"); |
ff7c6d11 | 216 | } else if let PassMode::Cast(cast) = self.mode { |
a7813a04 XL |
217 | // FIXME(eddyb): Figure out when the simpler Store is safe, clang |
218 | // uses it for i16 -> {i8, i8}, but not for i24 -> {i8, i8, i8}. | |
219 | let can_store_through_cast_ptr = false; | |
220 | if can_store_through_cast_ptr { | |
a1dfa0c6 XL |
221 | let cast_ptr_llty = bx.type_ptr_to(cast.llvm_type(bx)); |
222 | let cast_dst = bx.pointercast(dst.llval, cast_ptr_llty); | |
223 | bx.store(val, cast_dst, self.layout.align.abi); | |
a7813a04 XL |
224 | } else { |
225 | // The actual return type is a struct, but the ABI | |
226 | // adaptation code has cast it into some scalar type. The | |
227 | // code that follows is the only reliable way I have | |
228 | // found to do a transform like i64 -> {i32,i32}. | |
229 | // Basically we dump the data onto the stack then memcpy it. | |
230 | // | |
231 | // Other approaches I tried: | |
232 | // - Casting rust ret pointer to the foreign type and using Store | |
233 | // is (a) unsafe if size of foreign type > size of rust type and | |
234 | // (b) runs afoul of strict aliasing rules, yielding invalid | |
235 | // assembly under -O (specifically, the store gets removed). | |
236 | // - Truncating foreign type to correct integral type and then | |
237 | // bitcasting to the struct type yields invalid cast errors. | |
238 | ||
239 | // We instead thus allocate some scratch space... | |
a1dfa0c6 XL |
240 | let scratch_size = cast.size(bx); |
241 | let scratch_align = cast.align(bx); | |
e1599b0c | 242 | let llscratch = bx.alloca(cast.llvm_type(bx), scratch_align); |
2c00a5a8 | 243 | bx.lifetime_start(llscratch, scratch_size); |
a7813a04 | 244 | |
60c5eb7d | 245 | // ... where we first store the value... |
2c00a5a8 | 246 | bx.store(val, llscratch, scratch_align); |
a7813a04 | 247 | |
60c5eb7d | 248 | // ... and then memcpy it to the intended destination. |
a1dfa0c6 XL |
249 | bx.memcpy( |
250 | dst.llval, | |
251 | self.layout.align.abi, | |
252 | llscratch, | |
253 | scratch_align, | |
254 | bx.const_usize(self.layout.size.bytes()), | |
dfeec247 | 255 | MemFlags::empty(), |
a1dfa0c6 | 256 | ); |
a7813a04 | 257 | |
2c00a5a8 | 258 | bx.lifetime_end(llscratch, scratch_size); |
54a0048b SL |
259 | } |
260 | } else { | |
2c00a5a8 | 261 | OperandValue::Immediate(val).store(bx, dst); |
54a0048b SL |
262 | } |
263 | } | |
264 | ||
a1dfa0c6 XL |
265 | fn store_fn_arg( |
266 | &self, | |
a2a8927a | 267 | bx: &mut Builder<'_, 'll, 'tcx>, |
a1dfa0c6 XL |
268 | idx: &mut usize, |
269 | dst: PlaceRef<'tcx, &'ll Value>, | |
270 | ) { | |
ff7c6d11 | 271 | let mut next = || { |
2c00a5a8 | 272 | let val = llvm::get_param(bx.llfn(), *idx as c_uint); |
54a0048b | 273 | *idx += 1; |
ff7c6d11 XL |
274 | val |
275 | }; | |
276 | match self.mode { | |
e74abb32 | 277 | PassMode::Ignore => {} |
ff7c6d11 | 278 | PassMode::Pair(..) => { |
2c00a5a8 | 279 | OperandValue::Pair(next(), next()).store(bx, dst); |
ff7c6d11 | 280 | } |
fc512014 | 281 | PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => { |
a1dfa0c6 | 282 | OperandValue::Ref(next(), Some(next()), self.layout.align.abi).store(bx, dst); |
b7449926 | 283 | } |
fc512014 XL |
284 | PassMode::Direct(_) |
285 | | PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } | |
286 | | PassMode::Cast(_) => { | |
532ac7d7 XL |
287 | let next_arg = next(); |
288 | self.store(bx, next_arg, dst); | |
ff7c6d11 | 289 | } |
54a0048b | 290 | } |
54a0048b SL |
291 | } |
292 | } | |
293 | ||
a2a8927a | 294 | impl<'ll, 'tcx> ArgAbiMethods<'tcx> for Builder<'_, 'll, 'tcx> { |
a1dfa0c6 XL |
295 | fn store_fn_arg( |
296 | &mut self, | |
60c5eb7d | 297 | arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, |
dfeec247 XL |
298 | idx: &mut usize, |
299 | dst: PlaceRef<'tcx, Self::Value>, | |
a1dfa0c6 | 300 | ) { |
60c5eb7d | 301 | arg_abi.store_fn_arg(self, idx, dst) |
a1dfa0c6 | 302 | } |
60c5eb7d | 303 | fn store_arg( |
a1dfa0c6 | 304 | &mut self, |
60c5eb7d | 305 | arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, |
a1dfa0c6 | 306 | val: &'ll Value, |
dfeec247 | 307 | dst: PlaceRef<'tcx, &'ll Value>, |
a1dfa0c6 | 308 | ) { |
60c5eb7d | 309 | arg_abi.store(self, val, dst) |
a1dfa0c6 | 310 | } |
60c5eb7d XL |
311 | fn arg_memory_ty(&self, arg_abi: &ArgAbi<'tcx, Ty<'tcx>>) -> &'ll Type { |
312 | arg_abi.memory_ty(self) | |
a1dfa0c6 XL |
313 | } |
314 | } | |
315 | ||
a2a8927a | 316 | pub trait FnAbiLlvmExt<'ll, 'tcx> { |
b7449926 | 317 | fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type; |
a1dfa0c6 | 318 | fn ptr_to_llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type; |
83c7162d | 319 | fn llvm_cconv(&self) -> llvm::CallConv; |
416331ca | 320 | fn apply_attrs_llfn(&self, cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value); |
a2a8927a | 321 | fn apply_attrs_callsite(&self, bx: &mut Builder<'_, 'll, 'tcx>, callsite: &'ll Value); |
54a0048b SL |
322 | } |
323 | ||
a2a8927a | 324 | impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { |
b7449926 | 325 | fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type { |
94222f64 XL |
326 | // Ignore "extra" args from the call site for C variadic functions. |
327 | // Only the "fixed" args are part of the LLVM function signature. | |
328 | let args = if self.c_variadic { &self.args[..self.fixed_count] } else { &self.args }; | |
329 | ||
330 | let args_capacity: usize = args.iter().map(|arg| | |
8faf50e0 XL |
331 | if arg.pad.is_some() { 1 } else { 0 } + |
332 | if let PassMode::Pair(_, _) = arg.mode { 2 } else { 1 } | |
333 | ).sum(); | |
334 | let mut llargument_tys = Vec::with_capacity( | |
fc512014 | 335 | if let PassMode::Indirect { .. } = self.ret.mode { 1 } else { 0 } + args_capacity, |
8faf50e0 | 336 | ); |
54a0048b | 337 | |
ff7c6d11 | 338 | let llreturn_ty = match self.ret.mode { |
e74abb32 | 339 | PassMode::Ignore => cx.type_void(), |
dfeec247 | 340 | PassMode::Direct(_) | PassMode::Pair(..) => self.ret.layout.immediate_llvm_type(cx), |
2c00a5a8 | 341 | PassMode::Cast(cast) => cast.llvm_type(cx), |
fc512014 | 342 | PassMode::Indirect { .. } => { |
a1dfa0c6 XL |
343 | llargument_tys.push(cx.type_ptr_to(self.ret.memory_ty(cx))); |
344 | cx.type_void() | |
ff7c6d11 | 345 | } |
54a0048b SL |
346 | }; |
347 | ||
94222f64 | 348 | for arg in args { |
54a0048b SL |
349 | // add padding |
350 | if let Some(ty) = arg.pad { | |
2c00a5a8 | 351 | llargument_tys.push(ty.llvm_type(cx)); |
54a0048b SL |
352 | } |
353 | ||
ff7c6d11 | 354 | let llarg_ty = match arg.mode { |
e74abb32 | 355 | PassMode::Ignore => continue, |
2c00a5a8 | 356 | PassMode::Direct(_) => arg.layout.immediate_llvm_type(cx), |
ff7c6d11 | 357 | PassMode::Pair(..) => { |
8faf50e0 XL |
358 | llargument_tys.push(arg.layout.scalar_pair_element_llvm_type(cx, 0, true)); |
359 | llargument_tys.push(arg.layout.scalar_pair_element_llvm_type(cx, 1, true)); | |
ff7c6d11 XL |
360 | continue; |
361 | } | |
fc512014 | 362 | PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => { |
b7449926 XL |
363 | let ptr_ty = cx.tcx.mk_mut_ptr(arg.layout.ty); |
364 | let ptr_layout = cx.layout_of(ptr_ty); | |
365 | llargument_tys.push(ptr_layout.scalar_pair_element_llvm_type(cx, 0, true)); | |
366 | llargument_tys.push(ptr_layout.scalar_pair_element_llvm_type(cx, 1, true)); | |
367 | continue; | |
368 | } | |
2c00a5a8 | 369 | PassMode::Cast(cast) => cast.llvm_type(cx), |
fc512014 XL |
370 | PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } => { |
371 | cx.type_ptr_to(arg.memory_ty(cx)) | |
372 | } | |
54a0048b | 373 | }; |
54a0048b SL |
374 | llargument_tys.push(llarg_ty); |
375 | } | |
376 | ||
532ac7d7 | 377 | if self.c_variadic { |
a1dfa0c6 | 378 | cx.type_variadic_func(&llargument_tys, llreturn_ty) |
54a0048b | 379 | } else { |
a1dfa0c6 XL |
380 | cx.type_func(&llargument_tys, llreturn_ty) |
381 | } | |
382 | } | |
383 | ||
384 | fn ptr_to_llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type { | |
385 | unsafe { | |
dfeec247 XL |
386 | llvm::LLVMPointerType( |
387 | self.llvm_type(cx), | |
3dfed10e | 388 | cx.data_layout().instruction_address_space.0 as c_uint, |
dfeec247 | 389 | ) |
54a0048b SL |
390 | } |
391 | } | |
392 | ||
83c7162d XL |
393 | fn llvm_cconv(&self) -> llvm::CallConv { |
394 | match self.conv { | |
5869c6ff | 395 | Conv::C | Conv::Rust | Conv::CCmseNonSecureCall => llvm::CCallConv, |
923072b8 | 396 | Conv::RustCold => llvm::ColdCallConv, |
8faf50e0 | 397 | Conv::AmdGpuKernel => llvm::AmdGpuKernel, |
f035d41b XL |
398 | Conv::AvrInterrupt => llvm::AvrInterrupt, |
399 | Conv::AvrNonBlockingInterrupt => llvm::AvrNonBlockingInterrupt, | |
83c7162d XL |
400 | Conv::ArmAapcs => llvm::ArmAapcsCallConv, |
401 | Conv::Msp430Intr => llvm::Msp430Intr, | |
402 | Conv::PtxKernel => llvm::PtxKernel, | |
403 | Conv::X86Fastcall => llvm::X86FastcallCallConv, | |
404 | Conv::X86Intr => llvm::X86_Intr, | |
405 | Conv::X86Stdcall => llvm::X86StdcallCallConv, | |
406 | Conv::X86ThisCall => llvm::X86_ThisCall, | |
407 | Conv::X86VectorCall => llvm::X86_VectorCall, | |
408 | Conv::X86_64SysV => llvm::X86_64_SysV, | |
409 | Conv::X86_64Win64 => llvm::X86_64_Win64, | |
410 | } | |
411 | } | |
412 | ||
416331ca | 413 | fn apply_attrs_llfn(&self, cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value) { |
5e7ed085 | 414 | let mut func_attrs = SmallVec::<[_; 2]>::new(); |
60c5eb7d | 415 | if self.ret.layout.abi.is_uninhabited() { |
5e7ed085 | 416 | func_attrs.push(llvm::AttributeKind::NoReturn.create_attr(cx.llcx)); |
60c5eb7d | 417 | } |
ba9703b0 | 418 | if !self.can_unwind { |
5e7ed085 | 419 | func_attrs.push(llvm::AttributeKind::NoUnwind.create_attr(cx.llcx)); |
ba9703b0 | 420 | } |
5e7ed085 | 421 | attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &{ func_attrs }); |
ba9703b0 | 422 | |
ff7c6d11 | 423 | let mut i = 0; |
fc512014 | 424 | let mut apply = |attrs: &ArgAttributes| { |
cdc7bbd5 | 425 | attrs.apply_attrs_to_llfn(llvm::AttributePlace::Argument(i), cx, llfn); |
ff7c6d11 | 426 | i += 1; |
fc512014 | 427 | i - 1 |
ff7c6d11 XL |
428 | }; |
429 | match self.ret.mode { | |
430 | PassMode::Direct(ref attrs) => { | |
cdc7bbd5 | 431 | attrs.apply_attrs_to_llfn(llvm::AttributePlace::ReturnValue, cx, llfn); |
fc512014 XL |
432 | } |
433 | PassMode::Indirect { ref attrs, extra_attrs: _, on_stack } => { | |
434 | assert!(!on_stack); | |
435 | let i = apply(attrs); | |
5e7ed085 FG |
436 | let sret = llvm::CreateStructRetAttr(cx.llcx, self.ret.layout.llvm_type(cx)); |
437 | attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[sret]); | |
ff7c6d11 | 438 | } |
a2a8927a XL |
439 | PassMode::Cast(cast) => { |
440 | cast.attrs.apply_attrs_to_llfn(llvm::AttributePlace::ReturnValue, cx, llfn); | |
441 | } | |
ff7c6d11 | 442 | _ => {} |
54a0048b | 443 | } |
54a0048b | 444 | for arg in &self.args { |
ff7c6d11 | 445 | if arg.pad.is_some() { |
fc512014 | 446 | apply(&ArgAttributes::new()); |
ff7c6d11 XL |
447 | } |
448 | match arg.mode { | |
e74abb32 | 449 | PassMode::Ignore => {} |
fc512014 XL |
450 | PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: true } => { |
451 | let i = apply(attrs); | |
5e7ed085 FG |
452 | let byval = llvm::CreateByValAttr(cx.llcx, arg.layout.llvm_type(cx)); |
453 | attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[byval]); | |
dfeec247 | 454 | } |
fc512014 XL |
455 | PassMode::Direct(ref attrs) |
456 | | PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: false } => { | |
457 | apply(attrs); | |
458 | } | |
459 | PassMode::Indirect { ref attrs, extra_attrs: Some(ref extra_attrs), on_stack } => { | |
460 | assert!(!on_stack); | |
461 | apply(attrs); | |
462 | apply(extra_attrs); | |
b7449926 | 463 | } |
ff7c6d11 | 464 | PassMode::Pair(ref a, ref b) => { |
fc512014 XL |
465 | apply(a); |
466 | apply(b); | |
467 | } | |
a2a8927a XL |
468 | PassMode::Cast(cast) => { |
469 | apply(&cast.attrs); | |
ff7c6d11 | 470 | } |
54a0048b SL |
471 | } |
472 | } | |
473 | } | |
474 | ||
a2a8927a | 475 | fn apply_attrs_callsite(&self, bx: &mut Builder<'_, 'll, 'tcx>, callsite: &'ll Value) { |
5e7ed085 | 476 | let mut func_attrs = SmallVec::<[_; 2]>::new(); |
c295e0f8 | 477 | if self.ret.layout.abi.is_uninhabited() { |
5e7ed085 | 478 | func_attrs.push(llvm::AttributeKind::NoReturn.create_attr(bx.cx.llcx)); |
c295e0f8 XL |
479 | } |
480 | if !self.can_unwind { | |
5e7ed085 | 481 | func_attrs.push(llvm::AttributeKind::NoUnwind.create_attr(bx.cx.llcx)); |
c295e0f8 | 482 | } |
5e7ed085 | 483 | attributes::apply_to_callsite(callsite, llvm::AttributePlace::Function, &{ func_attrs }); |
ba9703b0 | 484 | |
ff7c6d11 | 485 | let mut i = 0; |
cdc7bbd5 XL |
486 | let mut apply = |cx: &CodegenCx<'_, '_>, attrs: &ArgAttributes| { |
487 | attrs.apply_attrs_to_callsite(llvm::AttributePlace::Argument(i), cx, callsite); | |
ff7c6d11 | 488 | i += 1; |
fc512014 | 489 | i - 1 |
ff7c6d11 XL |
490 | }; |
491 | match self.ret.mode { | |
492 | PassMode::Direct(ref attrs) => { | |
c295e0f8 | 493 | attrs.apply_attrs_to_callsite(llvm::AttributePlace::ReturnValue, bx.cx, callsite); |
fc512014 XL |
494 | } |
495 | PassMode::Indirect { ref attrs, extra_attrs: _, on_stack } => { | |
496 | assert!(!on_stack); | |
cdc7bbd5 | 497 | let i = apply(bx.cx, attrs); |
5e7ed085 FG |
498 | let sret = llvm::CreateStructRetAttr(bx.cx.llcx, self.ret.layout.llvm_type(bx)); |
499 | attributes::apply_to_callsite(callsite, llvm::AttributePlace::Argument(i), &[sret]); | |
ff7c6d11 | 500 | } |
a2a8927a XL |
501 | PassMode::Cast(cast) => { |
502 | cast.attrs.apply_attrs_to_callsite( | |
503 | llvm::AttributePlace::ReturnValue, | |
504 | &bx.cx, | |
505 | callsite, | |
506 | ); | |
507 | } | |
ff7c6d11 | 508 | _ => {} |
54a0048b | 509 | } |
c295e0f8 | 510 | if let abi::Abi::Scalar(scalar) = self.ret.layout.abi { |
83c7162d XL |
511 | // If the value is a boolean, the range is 0..2 and that ultimately |
512 | // become 0..0 when the type becomes i1, which would be rejected | |
513 | // by the LLVM verifier. | |
04454e1e | 514 | if let Int(..) = scalar.primitive() { |
c295e0f8 | 515 | if !scalar.is_bool() && !scalar.is_always_valid(bx) { |
04454e1e | 516 | bx.range_metadata(callsite, scalar.valid_range(bx)); |
83c7162d | 517 | } |
83c7162d XL |
518 | } |
519 | } | |
54a0048b | 520 | for arg in &self.args { |
ff7c6d11 | 521 | if arg.pad.is_some() { |
cdc7bbd5 | 522 | apply(bx.cx, &ArgAttributes::new()); |
ff7c6d11 XL |
523 | } |
524 | match arg.mode { | |
e74abb32 | 525 | PassMode::Ignore => {} |
fc512014 | 526 | PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: true } => { |
cdc7bbd5 | 527 | let i = apply(bx.cx, attrs); |
5e7ed085 FG |
528 | let byval = llvm::CreateByValAttr(bx.cx.llcx, arg.layout.llvm_type(bx)); |
529 | attributes::apply_to_callsite( | |
530 | callsite, | |
531 | llvm::AttributePlace::Argument(i), | |
532 | &[byval], | |
533 | ); | |
dfeec247 | 534 | } |
fc512014 XL |
535 | PassMode::Direct(ref attrs) |
536 | | PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: false } => { | |
cdc7bbd5 | 537 | apply(bx.cx, attrs); |
fc512014 XL |
538 | } |
539 | PassMode::Indirect { | |
540 | ref attrs, | |
541 | extra_attrs: Some(ref extra_attrs), | |
542 | on_stack: _, | |
543 | } => { | |
cdc7bbd5 XL |
544 | apply(bx.cx, attrs); |
545 | apply(bx.cx, extra_attrs); | |
b7449926 | 546 | } |
ff7c6d11 | 547 | PassMode::Pair(ref a, ref b) => { |
cdc7bbd5 XL |
548 | apply(bx.cx, a); |
549 | apply(bx.cx, b); | |
fc512014 | 550 | } |
a2a8927a XL |
551 | PassMode::Cast(cast) => { |
552 | apply(bx.cx, &cast.attrs); | |
ff7c6d11 | 553 | } |
54a0048b SL |
554 | } |
555 | } | |
556 | ||
83c7162d XL |
557 | let cconv = self.llvm_cconv(); |
558 | if cconv != llvm::CCallConv { | |
559 | llvm::SetInstructionCallConv(callsite, cconv); | |
54a0048b | 560 | } |
5869c6ff XL |
561 | |
562 | if self.conv == Conv::CCmseNonSecureCall { | |
563 | // This will probably get ignored on all targets but those supporting the TrustZone-M | |
564 | // extension (thumbv8m targets). | |
5e7ed085 FG |
565 | let cmse_nonsecure_call = llvm::CreateAttrString(bx.cx.llcx, "cmse_nonsecure_call"); |
566 | attributes::apply_to_callsite( | |
567 | callsite, | |
568 | llvm::AttributePlace::Function, | |
569 | &[cmse_nonsecure_call], | |
570 | ); | |
5869c6ff | 571 | } |
064997fb FG |
572 | |
573 | // Some intrinsics require that an elementtype attribute (with the pointee type of a | |
574 | // pointer argument) is added to the callsite. | |
575 | let element_type_index = unsafe { llvm::LLVMRustGetElementTypeArgIndex(callsite) }; | |
576 | if element_type_index >= 0 { | |
577 | let arg_ty = self.args[element_type_index as usize].layout.ty; | |
578 | let pointee_ty = arg_ty.builtin_deref(true).expect("Must be pointer argument").ty; | |
579 | let element_type_attr = unsafe { | |
580 | llvm::LLVMRustCreateElementTypeAttr(bx.llcx, bx.layout_of(pointee_ty).llvm_type(bx)) | |
581 | }; | |
582 | attributes::apply_to_callsite( | |
583 | callsite, | |
584 | llvm::AttributePlace::Argument(element_type_index as u32), | |
585 | &[element_type_attr], | |
586 | ); | |
587 | } | |
54a0048b SL |
588 | } |
589 | } | |
a1dfa0c6 | 590 | |
a2a8927a | 591 | impl<'tcx> AbiBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { |
dfeec247 | 592 | fn apply_attrs_callsite(&mut self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>, callsite: Self::Value) { |
60c5eb7d | 593 | fn_abi.apply_attrs_callsite(self, callsite) |
a1dfa0c6 | 594 | } |
532ac7d7 | 595 | |
3c0e092e | 596 | fn get_param(&mut self, index: usize) -> Self::Value { |
532ac7d7 XL |
597 | llvm::get_param(self.llfn(), index as c_uint) |
598 | } | |
a1dfa0c6 | 599 | } |