]>
Commit | Line | Data |
---|---|---|
9fa01778 XL |
1 | use crate::builder::Builder; |
2 | use crate::context::CodegenCx; | |
60c5eb7d | 3 | use crate::llvm::{self, AttributePlace}; |
9fa01778 | 4 | use crate::type_::Type; |
60c5eb7d | 5 | use crate::type_of::LayoutLlvmExt; |
9fa01778 | 6 | use crate::value::Value; |
60c5eb7d | 7 | |
dfeec247 XL |
8 | use rustc_codegen_ssa::mir::operand::OperandValue; |
9 | use rustc_codegen_ssa::mir::place::PlaceRef; | |
a1dfa0c6 | 10 | use rustc_codegen_ssa::traits::*; |
dfeec247 | 11 | use rustc_codegen_ssa::MemFlags; |
ba9703b0 | 12 | use rustc_middle::bug; |
c295e0f8 | 13 | use rustc_middle::ty::layout::LayoutOf; |
ba9703b0 XL |
14 | pub use rustc_middle::ty::layout::{FAT_PTR_ADDR, FAT_PTR_EXTRA}; |
15 | use rustc_middle::ty::Ty; | |
60c5eb7d | 16 | use rustc_target::abi::call::ArgAbi; |
83c7162d | 17 | pub use rustc_target::abi::call::*; |
c295e0f8 | 18 | use rustc_target::abi::{self, HasDataLayout, Int}; |
dfeec247 | 19 | pub use rustc_target::spec::abi::Abi; |
476ff2be | 20 | |
ba9703b0 XL |
21 | use libc::c_uint; |
22 | ||
476ff2be SL |
23 | macro_rules! for_each_kind { |
24 | ($flags: ident, $f: ident, $($kind: ident),+) => ({ | |
25 | $(if $flags.contains(ArgAttribute::$kind) { $f(llvm::Attribute::$kind) })+ | |
26 | }) | |
27 | } | |
28 | ||
83c7162d | 29 | trait ArgAttributeExt { |
dfeec247 XL |
30 | fn for_each_kind<F>(&self, f: F) |
31 | where | |
32 | F: FnMut(llvm::Attribute); | |
83c7162d XL |
33 | } |
34 | ||
35 | impl ArgAttributeExt for ArgAttribute { | |
dfeec247 XL |
36 | fn for_each_kind<F>(&self, mut f: F) |
37 | where | |
38 | F: FnMut(llvm::Attribute), | |
39 | { | |
5099ac24 | 40 | for_each_kind!(self, f, NoAlias, NoCapture, NonNull, ReadOnly, InReg, NoUndef) |
476ff2be SL |
41 | } |
42 | } | |
43 | ||
83c7162d | 44 | pub trait ArgAttributesExt { |
cdc7bbd5 XL |
45 | fn apply_attrs_to_llfn(&self, idx: AttributePlace, cx: &CodegenCx<'_, '_>, llfn: &Value); |
46 | fn apply_attrs_to_callsite( | |
47 | &self, | |
48 | idx: AttributePlace, | |
49 | cx: &CodegenCx<'_, '_>, | |
50 | callsite: &Value, | |
51 | ); | |
52 | } | |
53 | ||
54 | fn should_use_mutable_noalias(cx: &CodegenCx<'_, '_>) -> bool { | |
3c0e092e XL |
55 | // LLVM prior to version 12 had known miscompiles in the presence of |
56 | // noalias attributes (see #54878), but we don't support earlier | |
57 | // versions at all anymore. We now enable mutable noalias by default. | |
58 | cx.tcx.sess.opts.debugging_opts.mutable_noalias.unwrap_or(true) | |
476ff2be SL |
59 | } |
60 | ||
83c7162d | 61 | impl ArgAttributesExt for ArgAttributes { |
cdc7bbd5 | 62 | fn apply_attrs_to_llfn(&self, idx: AttributePlace, cx: &CodegenCx<'_, '_>, llfn: &Value) { |
ff7c6d11 | 63 | let mut regular = self.regular; |
476ff2be | 64 | unsafe { |
ff7c6d11 XL |
65 | let deref = self.pointee_size.bytes(); |
66 | if deref != 0 { | |
67 | if regular.contains(ArgAttribute::NonNull) { | |
dfeec247 | 68 | llvm::LLVMRustAddDereferenceableAttr(llfn, idx.as_uint(), deref); |
ff7c6d11 | 69 | } else { |
dfeec247 | 70 | llvm::LLVMRustAddDereferenceableOrNullAttr(llfn, idx.as_uint(), deref); |
ff7c6d11 XL |
71 | } |
72 | regular -= ArgAttribute::NonNull; | |
476ff2be | 73 | } |
ff7c6d11 | 74 | if let Some(align) = self.pointee_align { |
dfeec247 | 75 | llvm::LLVMRustAddAlignmentAttr(llfn, idx.as_uint(), align.bytes() as u32); |
ff7c6d11 XL |
76 | } |
77 | regular.for_each_kind(|attr| attr.apply_llfn(idx, llfn)); | |
cdc7bbd5 XL |
78 | if regular.contains(ArgAttribute::NoAliasMutRef) && should_use_mutable_noalias(cx) { |
79 | llvm::Attribute::NoAlias.apply_llfn(idx, llfn); | |
80 | } | |
fc512014 XL |
81 | match self.arg_ext { |
82 | ArgExtension::None => {} | |
83 | ArgExtension::Zext => { | |
84 | llvm::Attribute::ZExt.apply_llfn(idx, llfn); | |
85 | } | |
86 | ArgExtension::Sext => { | |
87 | llvm::Attribute::SExt.apply_llfn(idx, llfn); | |
88 | } | |
89 | } | |
476ff2be SL |
90 | } |
91 | } | |
92 | ||
cdc7bbd5 XL |
93 | fn apply_attrs_to_callsite( |
94 | &self, | |
95 | idx: AttributePlace, | |
96 | cx: &CodegenCx<'_, '_>, | |
97 | callsite: &Value, | |
98 | ) { | |
ff7c6d11 | 99 | let mut regular = self.regular; |
476ff2be | 100 | unsafe { |
ff7c6d11 XL |
101 | let deref = self.pointee_size.bytes(); |
102 | if deref != 0 { | |
103 | if regular.contains(ArgAttribute::NonNull) { | |
dfeec247 | 104 | llvm::LLVMRustAddDereferenceableCallSiteAttr(callsite, idx.as_uint(), deref); |
ff7c6d11 | 105 | } else { |
dfeec247 XL |
106 | llvm::LLVMRustAddDereferenceableOrNullCallSiteAttr( |
107 | callsite, | |
108 | idx.as_uint(), | |
109 | deref, | |
110 | ); | |
ff7c6d11 XL |
111 | } |
112 | regular -= ArgAttribute::NonNull; | |
113 | } | |
114 | if let Some(align) = self.pointee_align { | |
dfeec247 XL |
115 | llvm::LLVMRustAddAlignmentCallSiteAttr( |
116 | callsite, | |
117 | idx.as_uint(), | |
118 | align.bytes() as u32, | |
119 | ); | |
476ff2be | 120 | } |
ff7c6d11 | 121 | regular.for_each_kind(|attr| attr.apply_callsite(idx, callsite)); |
cdc7bbd5 XL |
122 | if regular.contains(ArgAttribute::NoAliasMutRef) && should_use_mutable_noalias(cx) { |
123 | llvm::Attribute::NoAlias.apply_callsite(idx, callsite); | |
124 | } | |
fc512014 XL |
125 | match self.arg_ext { |
126 | ArgExtension::None => {} | |
127 | ArgExtension::Zext => { | |
128 | llvm::Attribute::ZExt.apply_callsite(idx, callsite); | |
129 | } | |
130 | ArgExtension::Sext => { | |
131 | llvm::Attribute::SExt.apply_callsite(idx, callsite); | |
132 | } | |
133 | } | |
476ff2be SL |
134 | } |
135 | } | |
136 | } | |
cc61c64b | 137 | |
83c7162d | 138 | pub trait LlvmType { |
a2a8927a | 139 | fn llvm_type<'ll>(&self, cx: &CodegenCx<'ll, '_>) -> &'ll Type; |
cc61c64b XL |
140 | } |
141 | ||
83c7162d | 142 | impl LlvmType for Reg { |
a2a8927a | 143 | fn llvm_type<'ll>(&self, cx: &CodegenCx<'ll, '_>) -> &'ll Type { |
cc61c64b | 144 | match self.kind { |
a1dfa0c6 | 145 | RegKind::Integer => cx.type_ix(self.size.bits()), |
dfeec247 XL |
146 | RegKind::Float => match self.size.bits() { |
147 | 32 => cx.type_f32(), | |
148 | 64 => cx.type_f64(), | |
149 | _ => bug!("unsupported float: {:?}", self), | |
150 | }, | |
151 | RegKind::Vector => cx.type_vector(cx.type_i8(), self.size.bytes()), | |
cc61c64b XL |
152 | } |
153 | } | |
154 | } | |
155 | ||
83c7162d | 156 | impl LlvmType for CastTarget { |
a2a8927a | 157 | fn llvm_type<'ll>(&self, cx: &CodegenCx<'ll, '_>) -> &'ll Type { |
0531ce1d | 158 | let rest_ll_unit = self.rest.unit.llvm_type(cx); |
94b46f34 XL |
159 | let (rest_count, rem_bytes) = if self.rest.unit.size.bytes() == 0 { |
160 | (0, 0) | |
161 | } else { | |
dfeec247 XL |
162 | ( |
163 | self.rest.total.bytes() / self.rest.unit.size.bytes(), | |
164 | self.rest.total.bytes() % self.rest.unit.size.bytes(), | |
165 | ) | |
94b46f34 | 166 | }; |
0531ce1d XL |
167 | |
168 | if self.prefix.iter().all(|x| x.is_none()) { | |
169 | // Simplify to a single unit when there is no prefix and size <= unit size | |
170 | if self.rest.total <= self.rest.unit.size { | |
171 | return rest_ll_unit; | |
172 | } | |
173 | ||
174 | // Simplify to array when all chunks are the same size and type | |
175 | if rem_bytes == 0 { | |
a1dfa0c6 | 176 | return cx.type_array(rest_ll_unit, rest_count); |
cc61c64b XL |
177 | } |
178 | } | |
0531ce1d XL |
179 | |
180 | // Create list of fields in the main structure | |
dfeec247 XL |
181 | let mut args: Vec<_> = self |
182 | .prefix | |
183 | .iter() | |
a2a8927a | 184 | .flat_map(|option_reg| option_reg.map(|reg| reg.llvm_type(cx))) |
0531ce1d XL |
185 | .chain((0..rest_count).map(|_| rest_ll_unit)) |
186 | .collect(); | |
187 | ||
188 | // Append final integer | |
189 | if rem_bytes != 0 { | |
190 | // Only integers can be really split further. | |
191 | assert_eq!(self.rest.unit.kind, RegKind::Integer); | |
a1dfa0c6 | 192 | args.push(cx.type_ix(rem_bytes * 8)); |
0531ce1d XL |
193 | } |
194 | ||
a1dfa0c6 | 195 | cx.type_struct(&args, false) |
cc61c64b XL |
196 | } |
197 | } | |
476ff2be | 198 | |
60c5eb7d | 199 | pub trait ArgAbiExt<'ll, 'tcx> { |
b7449926 | 200 | fn memory_ty(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type; |
a1dfa0c6 XL |
201 | fn store( |
202 | &self, | |
203 | bx: &mut Builder<'_, 'll, 'tcx>, | |
204 | val: &'ll Value, | |
205 | dst: PlaceRef<'tcx, &'ll Value>, | |
206 | ); | |
207 | fn store_fn_arg( | |
208 | &self, | |
209 | bx: &mut Builder<'_, 'll, 'tcx>, | |
210 | idx: &mut usize, | |
211 | dst: PlaceRef<'tcx, &'ll Value>, | |
212 | ); | |
54a0048b SL |
213 | } |
214 | ||
a2a8927a | 215 | impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> { |
9fa01778 | 216 | /// Gets the LLVM type for a place of the original Rust type of |
0731742a | 217 | /// this argument/return, i.e., the result of `type_of::type_of`. |
b7449926 | 218 | fn memory_ty(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type { |
2c00a5a8 | 219 | self.layout.llvm_type(cx) |
54a0048b SL |
220 | } |
221 | ||
60c5eb7d | 222 | /// Stores a direct/indirect value described by this ArgAbi into a |
ff7c6d11 | 223 | /// place for the original Rust type of this argument/return. |
54a0048b SL |
224 | /// Can be used for both storing formal arguments into Rust variables |
225 | /// or results of call/invoke instructions into their destinations. | |
a1dfa0c6 XL |
226 | fn store( |
227 | &self, | |
228 | bx: &mut Builder<'_, 'll, 'tcx>, | |
229 | val: &'ll Value, | |
230 | dst: PlaceRef<'tcx, &'ll Value>, | |
231 | ) { | |
54a0048b SL |
232 | if self.is_ignore() { |
233 | return; | |
234 | } | |
b7449926 | 235 | if self.is_sized_indirect() { |
a1dfa0c6 | 236 | OperandValue::Ref(val, None, self.layout.align.abi).store(bx, dst) |
b7449926 | 237 | } else if self.is_unsized_indirect() { |
60c5eb7d | 238 | bug!("unsized `ArgAbi` must be handled through `store_fn_arg`"); |
ff7c6d11 | 239 | } else if let PassMode::Cast(cast) = self.mode { |
a7813a04 XL |
240 | // FIXME(eddyb): Figure out when the simpler Store is safe, clang |
241 | // uses it for i16 -> {i8, i8}, but not for i24 -> {i8, i8, i8}. | |
242 | let can_store_through_cast_ptr = false; | |
243 | if can_store_through_cast_ptr { | |
a1dfa0c6 XL |
244 | let cast_ptr_llty = bx.type_ptr_to(cast.llvm_type(bx)); |
245 | let cast_dst = bx.pointercast(dst.llval, cast_ptr_llty); | |
246 | bx.store(val, cast_dst, self.layout.align.abi); | |
a7813a04 XL |
247 | } else { |
248 | // The actual return type is a struct, but the ABI | |
249 | // adaptation code has cast it into some scalar type. The | |
250 | // code that follows is the only reliable way I have | |
251 | // found to do a transform like i64 -> {i32,i32}. | |
252 | // Basically we dump the data onto the stack then memcpy it. | |
253 | // | |
254 | // Other approaches I tried: | |
255 | // - Casting rust ret pointer to the foreign type and using Store | |
256 | // is (a) unsafe if size of foreign type > size of rust type and | |
257 | // (b) runs afoul of strict aliasing rules, yielding invalid | |
258 | // assembly under -O (specifically, the store gets removed). | |
259 | // - Truncating foreign type to correct integral type and then | |
260 | // bitcasting to the struct type yields invalid cast errors. | |
261 | ||
262 | // We instead thus allocate some scratch space... | |
a1dfa0c6 XL |
263 | let scratch_size = cast.size(bx); |
264 | let scratch_align = cast.align(bx); | |
e1599b0c | 265 | let llscratch = bx.alloca(cast.llvm_type(bx), scratch_align); |
2c00a5a8 | 266 | bx.lifetime_start(llscratch, scratch_size); |
a7813a04 | 267 | |
60c5eb7d | 268 | // ... where we first store the value... |
2c00a5a8 | 269 | bx.store(val, llscratch, scratch_align); |
a7813a04 | 270 | |
60c5eb7d | 271 | // ... and then memcpy it to the intended destination. |
a1dfa0c6 XL |
272 | bx.memcpy( |
273 | dst.llval, | |
274 | self.layout.align.abi, | |
275 | llscratch, | |
276 | scratch_align, | |
277 | bx.const_usize(self.layout.size.bytes()), | |
dfeec247 | 278 | MemFlags::empty(), |
a1dfa0c6 | 279 | ); |
a7813a04 | 280 | |
2c00a5a8 | 281 | bx.lifetime_end(llscratch, scratch_size); |
54a0048b SL |
282 | } |
283 | } else { | |
2c00a5a8 | 284 | OperandValue::Immediate(val).store(bx, dst); |
54a0048b SL |
285 | } |
286 | } | |
287 | ||
a1dfa0c6 XL |
288 | fn store_fn_arg( |
289 | &self, | |
a2a8927a | 290 | bx: &mut Builder<'_, 'll, 'tcx>, |
a1dfa0c6 XL |
291 | idx: &mut usize, |
292 | dst: PlaceRef<'tcx, &'ll Value>, | |
293 | ) { | |
ff7c6d11 | 294 | let mut next = || { |
2c00a5a8 | 295 | let val = llvm::get_param(bx.llfn(), *idx as c_uint); |
54a0048b | 296 | *idx += 1; |
ff7c6d11 XL |
297 | val |
298 | }; | |
299 | match self.mode { | |
e74abb32 | 300 | PassMode::Ignore => {} |
ff7c6d11 | 301 | PassMode::Pair(..) => { |
2c00a5a8 | 302 | OperandValue::Pair(next(), next()).store(bx, dst); |
ff7c6d11 | 303 | } |
fc512014 | 304 | PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => { |
a1dfa0c6 | 305 | OperandValue::Ref(next(), Some(next()), self.layout.align.abi).store(bx, dst); |
b7449926 | 306 | } |
fc512014 XL |
307 | PassMode::Direct(_) |
308 | | PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } | |
309 | | PassMode::Cast(_) => { | |
532ac7d7 XL |
310 | let next_arg = next(); |
311 | self.store(bx, next_arg, dst); | |
ff7c6d11 | 312 | } |
54a0048b | 313 | } |
54a0048b SL |
314 | } |
315 | } | |
316 | ||
a2a8927a | 317 | impl<'ll, 'tcx> ArgAbiMethods<'tcx> for Builder<'_, 'll, 'tcx> { |
a1dfa0c6 XL |
318 | fn store_fn_arg( |
319 | &mut self, | |
60c5eb7d | 320 | arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, |
dfeec247 XL |
321 | idx: &mut usize, |
322 | dst: PlaceRef<'tcx, Self::Value>, | |
a1dfa0c6 | 323 | ) { |
60c5eb7d | 324 | arg_abi.store_fn_arg(self, idx, dst) |
a1dfa0c6 | 325 | } |
60c5eb7d | 326 | fn store_arg( |
a1dfa0c6 | 327 | &mut self, |
60c5eb7d | 328 | arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, |
a1dfa0c6 | 329 | val: &'ll Value, |
dfeec247 | 330 | dst: PlaceRef<'tcx, &'ll Value>, |
a1dfa0c6 | 331 | ) { |
60c5eb7d | 332 | arg_abi.store(self, val, dst) |
a1dfa0c6 | 333 | } |
60c5eb7d XL |
334 | fn arg_memory_ty(&self, arg_abi: &ArgAbi<'tcx, Ty<'tcx>>) -> &'ll Type { |
335 | arg_abi.memory_ty(self) | |
a1dfa0c6 XL |
336 | } |
337 | } | |
338 | ||
a2a8927a | 339 | pub trait FnAbiLlvmExt<'ll, 'tcx> { |
b7449926 | 340 | fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type; |
a1dfa0c6 | 341 | fn ptr_to_llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type; |
83c7162d | 342 | fn llvm_cconv(&self) -> llvm::CallConv; |
416331ca | 343 | fn apply_attrs_llfn(&self, cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value); |
a2a8927a | 344 | fn apply_attrs_callsite(&self, bx: &mut Builder<'_, 'll, 'tcx>, callsite: &'ll Value); |
54a0048b SL |
345 | } |
346 | ||
a2a8927a | 347 | impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { |
b7449926 | 348 | fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type { |
94222f64 XL |
349 | // Ignore "extra" args from the call site for C variadic functions. |
350 | // Only the "fixed" args are part of the LLVM function signature. | |
351 | let args = if self.c_variadic { &self.args[..self.fixed_count] } else { &self.args }; | |
352 | ||
353 | let args_capacity: usize = args.iter().map(|arg| | |
8faf50e0 XL |
354 | if arg.pad.is_some() { 1 } else { 0 } + |
355 | if let PassMode::Pair(_, _) = arg.mode { 2 } else { 1 } | |
356 | ).sum(); | |
357 | let mut llargument_tys = Vec::with_capacity( | |
fc512014 | 358 | if let PassMode::Indirect { .. } = self.ret.mode { 1 } else { 0 } + args_capacity, |
8faf50e0 | 359 | ); |
54a0048b | 360 | |
ff7c6d11 | 361 | let llreturn_ty = match self.ret.mode { |
e74abb32 | 362 | PassMode::Ignore => cx.type_void(), |
dfeec247 | 363 | PassMode::Direct(_) | PassMode::Pair(..) => self.ret.layout.immediate_llvm_type(cx), |
2c00a5a8 | 364 | PassMode::Cast(cast) => cast.llvm_type(cx), |
fc512014 | 365 | PassMode::Indirect { .. } => { |
a1dfa0c6 XL |
366 | llargument_tys.push(cx.type_ptr_to(self.ret.memory_ty(cx))); |
367 | cx.type_void() | |
ff7c6d11 | 368 | } |
54a0048b SL |
369 | }; |
370 | ||
94222f64 | 371 | for arg in args { |
54a0048b SL |
372 | // add padding |
373 | if let Some(ty) = arg.pad { | |
2c00a5a8 | 374 | llargument_tys.push(ty.llvm_type(cx)); |
54a0048b SL |
375 | } |
376 | ||
ff7c6d11 | 377 | let llarg_ty = match arg.mode { |
e74abb32 | 378 | PassMode::Ignore => continue, |
2c00a5a8 | 379 | PassMode::Direct(_) => arg.layout.immediate_llvm_type(cx), |
ff7c6d11 | 380 | PassMode::Pair(..) => { |
8faf50e0 XL |
381 | llargument_tys.push(arg.layout.scalar_pair_element_llvm_type(cx, 0, true)); |
382 | llargument_tys.push(arg.layout.scalar_pair_element_llvm_type(cx, 1, true)); | |
ff7c6d11 XL |
383 | continue; |
384 | } | |
fc512014 | 385 | PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => { |
b7449926 XL |
386 | let ptr_ty = cx.tcx.mk_mut_ptr(arg.layout.ty); |
387 | let ptr_layout = cx.layout_of(ptr_ty); | |
388 | llargument_tys.push(ptr_layout.scalar_pair_element_llvm_type(cx, 0, true)); | |
389 | llargument_tys.push(ptr_layout.scalar_pair_element_llvm_type(cx, 1, true)); | |
390 | continue; | |
391 | } | |
2c00a5a8 | 392 | PassMode::Cast(cast) => cast.llvm_type(cx), |
fc512014 XL |
393 | PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } => { |
394 | cx.type_ptr_to(arg.memory_ty(cx)) | |
395 | } | |
54a0048b | 396 | }; |
54a0048b SL |
397 | llargument_tys.push(llarg_ty); |
398 | } | |
399 | ||
532ac7d7 | 400 | if self.c_variadic { |
a1dfa0c6 | 401 | cx.type_variadic_func(&llargument_tys, llreturn_ty) |
54a0048b | 402 | } else { |
a1dfa0c6 XL |
403 | cx.type_func(&llargument_tys, llreturn_ty) |
404 | } | |
405 | } | |
406 | ||
407 | fn ptr_to_llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type { | |
408 | unsafe { | |
dfeec247 XL |
409 | llvm::LLVMPointerType( |
410 | self.llvm_type(cx), | |
3dfed10e | 411 | cx.data_layout().instruction_address_space.0 as c_uint, |
dfeec247 | 412 | ) |
54a0048b SL |
413 | } |
414 | } | |
415 | ||
83c7162d XL |
416 | fn llvm_cconv(&self) -> llvm::CallConv { |
417 | match self.conv { | |
5869c6ff | 418 | Conv::C | Conv::Rust | Conv::CCmseNonSecureCall => llvm::CCallConv, |
8faf50e0 | 419 | Conv::AmdGpuKernel => llvm::AmdGpuKernel, |
f035d41b XL |
420 | Conv::AvrInterrupt => llvm::AvrInterrupt, |
421 | Conv::AvrNonBlockingInterrupt => llvm::AvrNonBlockingInterrupt, | |
83c7162d XL |
422 | Conv::ArmAapcs => llvm::ArmAapcsCallConv, |
423 | Conv::Msp430Intr => llvm::Msp430Intr, | |
424 | Conv::PtxKernel => llvm::PtxKernel, | |
425 | Conv::X86Fastcall => llvm::X86FastcallCallConv, | |
426 | Conv::X86Intr => llvm::X86_Intr, | |
427 | Conv::X86Stdcall => llvm::X86StdcallCallConv, | |
428 | Conv::X86ThisCall => llvm::X86_ThisCall, | |
429 | Conv::X86VectorCall => llvm::X86_VectorCall, | |
430 | Conv::X86_64SysV => llvm::X86_64_SysV, | |
431 | Conv::X86_64Win64 => llvm::X86_64_Win64, | |
432 | } | |
433 | } | |
434 | ||
416331ca | 435 | fn apply_attrs_llfn(&self, cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value) { |
60c5eb7d XL |
436 | // FIXME(eddyb) can this also be applied to callsites? |
437 | if self.ret.layout.abi.is_uninhabited() { | |
438 | llvm::Attribute::NoReturn.apply_llfn(llvm::AttributePlace::Function, llfn); | |
439 | } | |
440 | ||
ba9703b0 XL |
441 | // FIXME(eddyb, wesleywiser): apply this to callsites as well? |
442 | if !self.can_unwind { | |
443 | llvm::Attribute::NoUnwind.apply_llfn(llvm::AttributePlace::Function, llfn); | |
444 | } | |
445 | ||
ff7c6d11 | 446 | let mut i = 0; |
fc512014 | 447 | let mut apply = |attrs: &ArgAttributes| { |
cdc7bbd5 | 448 | attrs.apply_attrs_to_llfn(llvm::AttributePlace::Argument(i), cx, llfn); |
ff7c6d11 | 449 | i += 1; |
fc512014 | 450 | i - 1 |
ff7c6d11 XL |
451 | }; |
452 | match self.ret.mode { | |
453 | PassMode::Direct(ref attrs) => { | |
cdc7bbd5 | 454 | attrs.apply_attrs_to_llfn(llvm::AttributePlace::ReturnValue, cx, llfn); |
fc512014 XL |
455 | } |
456 | PassMode::Indirect { ref attrs, extra_attrs: _, on_stack } => { | |
457 | assert!(!on_stack); | |
458 | let i = apply(attrs); | |
6a06907d XL |
459 | unsafe { |
460 | llvm::LLVMRustAddStructRetAttr( | |
461 | llfn, | |
462 | llvm::AttributePlace::Argument(i).as_uint(), | |
463 | self.ret.layout.llvm_type(cx), | |
464 | ); | |
465 | } | |
ff7c6d11 | 466 | } |
a2a8927a XL |
467 | PassMode::Cast(cast) => { |
468 | cast.attrs.apply_attrs_to_llfn(llvm::AttributePlace::ReturnValue, cx, llfn); | |
469 | } | |
ff7c6d11 | 470 | _ => {} |
54a0048b | 471 | } |
54a0048b | 472 | for arg in &self.args { |
ff7c6d11 | 473 | if arg.pad.is_some() { |
fc512014 | 474 | apply(&ArgAttributes::new()); |
ff7c6d11 XL |
475 | } |
476 | match arg.mode { | |
e74abb32 | 477 | PassMode::Ignore => {} |
fc512014 XL |
478 | PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: true } => { |
479 | let i = apply(attrs); | |
480 | unsafe { | |
481 | llvm::LLVMRustAddByValAttr( | |
482 | llfn, | |
483 | llvm::AttributePlace::Argument(i).as_uint(), | |
484 | arg.layout.llvm_type(cx), | |
485 | ); | |
486 | } | |
dfeec247 | 487 | } |
fc512014 XL |
488 | PassMode::Direct(ref attrs) |
489 | | PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: false } => { | |
490 | apply(attrs); | |
491 | } | |
492 | PassMode::Indirect { ref attrs, extra_attrs: Some(ref extra_attrs), on_stack } => { | |
493 | assert!(!on_stack); | |
494 | apply(attrs); | |
495 | apply(extra_attrs); | |
b7449926 | 496 | } |
ff7c6d11 | 497 | PassMode::Pair(ref a, ref b) => { |
fc512014 XL |
498 | apply(a); |
499 | apply(b); | |
500 | } | |
a2a8927a XL |
501 | PassMode::Cast(cast) => { |
502 | apply(&cast.attrs); | |
ff7c6d11 | 503 | } |
54a0048b SL |
504 | } |
505 | } | |
506 | } | |
507 | ||
a2a8927a | 508 | fn apply_attrs_callsite(&self, bx: &mut Builder<'_, 'll, 'tcx>, callsite: &'ll Value) { |
c295e0f8 XL |
509 | if self.ret.layout.abi.is_uninhabited() { |
510 | llvm::Attribute::NoReturn.apply_callsite(llvm::AttributePlace::Function, callsite); | |
511 | } | |
512 | if !self.can_unwind { | |
513 | llvm::Attribute::NoUnwind.apply_callsite(llvm::AttributePlace::Function, callsite); | |
514 | } | |
ba9703b0 | 515 | |
ff7c6d11 | 516 | let mut i = 0; |
cdc7bbd5 XL |
517 | let mut apply = |cx: &CodegenCx<'_, '_>, attrs: &ArgAttributes| { |
518 | attrs.apply_attrs_to_callsite(llvm::AttributePlace::Argument(i), cx, callsite); | |
ff7c6d11 | 519 | i += 1; |
fc512014 | 520 | i - 1 |
ff7c6d11 XL |
521 | }; |
522 | match self.ret.mode { | |
523 | PassMode::Direct(ref attrs) => { | |
c295e0f8 | 524 | attrs.apply_attrs_to_callsite(llvm::AttributePlace::ReturnValue, bx.cx, callsite); |
fc512014 XL |
525 | } |
526 | PassMode::Indirect { ref attrs, extra_attrs: _, on_stack } => { | |
527 | assert!(!on_stack); | |
cdc7bbd5 | 528 | let i = apply(bx.cx, attrs); |
6a06907d XL |
529 | unsafe { |
530 | llvm::LLVMRustAddStructRetCallSiteAttr( | |
531 | callsite, | |
532 | llvm::AttributePlace::Argument(i).as_uint(), | |
533 | self.ret.layout.llvm_type(bx), | |
534 | ); | |
535 | } | |
ff7c6d11 | 536 | } |
a2a8927a XL |
537 | PassMode::Cast(cast) => { |
538 | cast.attrs.apply_attrs_to_callsite( | |
539 | llvm::AttributePlace::ReturnValue, | |
540 | &bx.cx, | |
541 | callsite, | |
542 | ); | |
543 | } | |
ff7c6d11 | 544 | _ => {} |
54a0048b | 545 | } |
c295e0f8 | 546 | if let abi::Abi::Scalar(scalar) = self.ret.layout.abi { |
83c7162d XL |
547 | // If the value is a boolean, the range is 0..2 and that ultimately |
548 | // become 0..0 when the type becomes i1, which would be rejected | |
549 | // by the LLVM verifier. | |
ba9703b0 | 550 | if let Int(..) = scalar.value { |
c295e0f8 XL |
551 | if !scalar.is_bool() && !scalar.is_always_valid(bx) { |
552 | bx.range_metadata(callsite, scalar.valid_range); | |
83c7162d | 553 | } |
83c7162d XL |
554 | } |
555 | } | |
54a0048b | 556 | for arg in &self.args { |
ff7c6d11 | 557 | if arg.pad.is_some() { |
cdc7bbd5 | 558 | apply(bx.cx, &ArgAttributes::new()); |
ff7c6d11 XL |
559 | } |
560 | match arg.mode { | |
e74abb32 | 561 | PassMode::Ignore => {} |
fc512014 | 562 | PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: true } => { |
cdc7bbd5 | 563 | let i = apply(bx.cx, attrs); |
fc512014 XL |
564 | unsafe { |
565 | llvm::LLVMRustAddByValCallSiteAttr( | |
566 | callsite, | |
567 | llvm::AttributePlace::Argument(i).as_uint(), | |
568 | arg.layout.llvm_type(bx), | |
569 | ); | |
570 | } | |
dfeec247 | 571 | } |
fc512014 XL |
572 | PassMode::Direct(ref attrs) |
573 | | PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: false } => { | |
cdc7bbd5 | 574 | apply(bx.cx, attrs); |
fc512014 XL |
575 | } |
576 | PassMode::Indirect { | |
577 | ref attrs, | |
578 | extra_attrs: Some(ref extra_attrs), | |
579 | on_stack: _, | |
580 | } => { | |
cdc7bbd5 XL |
581 | apply(bx.cx, attrs); |
582 | apply(bx.cx, extra_attrs); | |
b7449926 | 583 | } |
ff7c6d11 | 584 | PassMode::Pair(ref a, ref b) => { |
cdc7bbd5 XL |
585 | apply(bx.cx, a); |
586 | apply(bx.cx, b); | |
fc512014 | 587 | } |
a2a8927a XL |
588 | PassMode::Cast(cast) => { |
589 | apply(bx.cx, &cast.attrs); | |
ff7c6d11 | 590 | } |
54a0048b SL |
591 | } |
592 | } | |
593 | ||
83c7162d XL |
594 | let cconv = self.llvm_cconv(); |
595 | if cconv != llvm::CCallConv { | |
596 | llvm::SetInstructionCallConv(callsite, cconv); | |
54a0048b | 597 | } |
5869c6ff XL |
598 | |
599 | if self.conv == Conv::CCmseNonSecureCall { | |
600 | // This will probably get ignored on all targets but those supporting the TrustZone-M | |
601 | // extension (thumbv8m targets). | |
602 | unsafe { | |
603 | llvm::AddCallSiteAttrString( | |
604 | callsite, | |
605 | llvm::AttributePlace::Function, | |
6a06907d | 606 | cstr::cstr!("cmse_nonsecure_call"), |
5869c6ff XL |
607 | ); |
608 | } | |
609 | } | |
54a0048b SL |
610 | } |
611 | } | |
a1dfa0c6 | 612 | |
a2a8927a | 613 | impl<'tcx> AbiBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { |
dfeec247 | 614 | fn apply_attrs_callsite(&mut self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>, callsite: Self::Value) { |
60c5eb7d | 615 | fn_abi.apply_attrs_callsite(self, callsite) |
a1dfa0c6 | 616 | } |
532ac7d7 | 617 | |
3c0e092e | 618 | fn get_param(&mut self, index: usize) -> Self::Value { |
532ac7d7 XL |
619 | llvm::get_param(self.llfn(), index as c_uint) |
620 | } | |
a1dfa0c6 | 621 | } |