]>
Commit | Line | Data |
---|---|---|
9fa01778 XL |
1 | use crate::builder::Builder; |
2 | use crate::context::CodegenCx; | |
60c5eb7d | 3 | use crate::llvm::{self, AttributePlace}; |
9fa01778 | 4 | use crate::type_::Type; |
60c5eb7d | 5 | use crate::type_of::LayoutLlvmExt; |
9fa01778 | 6 | use crate::value::Value; |
60c5eb7d | 7 | |
dfeec247 XL |
8 | use rustc_codegen_ssa::mir::operand::OperandValue; |
9 | use rustc_codegen_ssa::mir::place::PlaceRef; | |
a1dfa0c6 | 10 | use rustc_codegen_ssa::traits::*; |
dfeec247 | 11 | use rustc_codegen_ssa::MemFlags; |
ba9703b0 | 12 | use rustc_middle::bug; |
c295e0f8 | 13 | use rustc_middle::ty::layout::LayoutOf; |
ba9703b0 XL |
14 | pub use rustc_middle::ty::layout::{FAT_PTR_ADDR, FAT_PTR_EXTRA}; |
15 | use rustc_middle::ty::Ty; | |
60c5eb7d | 16 | use rustc_target::abi::call::ArgAbi; |
83c7162d | 17 | pub use rustc_target::abi::call::*; |
c295e0f8 | 18 | use rustc_target::abi::{self, HasDataLayout, Int}; |
dfeec247 | 19 | pub use rustc_target::spec::abi::Abi; |
476ff2be | 20 | |
ba9703b0 XL |
21 | use libc::c_uint; |
22 | ||
476ff2be SL |
23 | macro_rules! for_each_kind { |
24 | ($flags: ident, $f: ident, $($kind: ident),+) => ({ | |
25 | $(if $flags.contains(ArgAttribute::$kind) { $f(llvm::Attribute::$kind) })+ | |
26 | }) | |
27 | } | |
28 | ||
83c7162d | 29 | trait ArgAttributeExt { |
dfeec247 XL |
30 | fn for_each_kind<F>(&self, f: F) |
31 | where | |
32 | F: FnMut(llvm::Attribute); | |
83c7162d XL |
33 | } |
34 | ||
35 | impl ArgAttributeExt for ArgAttribute { | |
dfeec247 XL |
36 | fn for_each_kind<F>(&self, mut f: F) |
37 | where | |
38 | F: FnMut(llvm::Attribute), | |
39 | { | |
fc512014 | 40 | for_each_kind!(self, f, NoAlias, NoCapture, NonNull, ReadOnly, InReg) |
476ff2be SL |
41 | } |
42 | } | |
43 | ||
83c7162d | 44 | pub trait ArgAttributesExt { |
cdc7bbd5 XL |
45 | fn apply_attrs_to_llfn(&self, idx: AttributePlace, cx: &CodegenCx<'_, '_>, llfn: &Value); |
46 | fn apply_attrs_to_callsite( | |
47 | &self, | |
48 | idx: AttributePlace, | |
49 | cx: &CodegenCx<'_, '_>, | |
50 | callsite: &Value, | |
51 | ); | |
52 | } | |
53 | ||
54 | fn should_use_mutable_noalias(cx: &CodegenCx<'_, '_>) -> bool { | |
3c0e092e XL |
55 | // LLVM prior to version 12 had known miscompiles in the presence of |
56 | // noalias attributes (see #54878), but we don't support earlier | |
57 | // versions at all anymore. We now enable mutable noalias by default. | |
58 | cx.tcx.sess.opts.debugging_opts.mutable_noalias.unwrap_or(true) | |
476ff2be SL |
59 | } |
60 | ||
83c7162d | 61 | impl ArgAttributesExt for ArgAttributes { |
cdc7bbd5 | 62 | fn apply_attrs_to_llfn(&self, idx: AttributePlace, cx: &CodegenCx<'_, '_>, llfn: &Value) { |
ff7c6d11 | 63 | let mut regular = self.regular; |
476ff2be | 64 | unsafe { |
ff7c6d11 XL |
65 | let deref = self.pointee_size.bytes(); |
66 | if deref != 0 { | |
67 | if regular.contains(ArgAttribute::NonNull) { | |
dfeec247 | 68 | llvm::LLVMRustAddDereferenceableAttr(llfn, idx.as_uint(), deref); |
ff7c6d11 | 69 | } else { |
dfeec247 | 70 | llvm::LLVMRustAddDereferenceableOrNullAttr(llfn, idx.as_uint(), deref); |
ff7c6d11 XL |
71 | } |
72 | regular -= ArgAttribute::NonNull; | |
476ff2be | 73 | } |
ff7c6d11 | 74 | if let Some(align) = self.pointee_align { |
dfeec247 | 75 | llvm::LLVMRustAddAlignmentAttr(llfn, idx.as_uint(), align.bytes() as u32); |
ff7c6d11 XL |
76 | } |
77 | regular.for_each_kind(|attr| attr.apply_llfn(idx, llfn)); | |
cdc7bbd5 XL |
78 | if regular.contains(ArgAttribute::NoAliasMutRef) && should_use_mutable_noalias(cx) { |
79 | llvm::Attribute::NoAlias.apply_llfn(idx, llfn); | |
80 | } | |
fc512014 XL |
81 | match self.arg_ext { |
82 | ArgExtension::None => {} | |
83 | ArgExtension::Zext => { | |
84 | llvm::Attribute::ZExt.apply_llfn(idx, llfn); | |
85 | } | |
86 | ArgExtension::Sext => { | |
87 | llvm::Attribute::SExt.apply_llfn(idx, llfn); | |
88 | } | |
89 | } | |
476ff2be SL |
90 | } |
91 | } | |
92 | ||
cdc7bbd5 XL |
93 | fn apply_attrs_to_callsite( |
94 | &self, | |
95 | idx: AttributePlace, | |
96 | cx: &CodegenCx<'_, '_>, | |
97 | callsite: &Value, | |
98 | ) { | |
ff7c6d11 | 99 | let mut regular = self.regular; |
476ff2be | 100 | unsafe { |
ff7c6d11 XL |
101 | let deref = self.pointee_size.bytes(); |
102 | if deref != 0 { | |
103 | if regular.contains(ArgAttribute::NonNull) { | |
dfeec247 | 104 | llvm::LLVMRustAddDereferenceableCallSiteAttr(callsite, idx.as_uint(), deref); |
ff7c6d11 | 105 | } else { |
dfeec247 XL |
106 | llvm::LLVMRustAddDereferenceableOrNullCallSiteAttr( |
107 | callsite, | |
108 | idx.as_uint(), | |
109 | deref, | |
110 | ); | |
ff7c6d11 XL |
111 | } |
112 | regular -= ArgAttribute::NonNull; | |
113 | } | |
114 | if let Some(align) = self.pointee_align { | |
dfeec247 XL |
115 | llvm::LLVMRustAddAlignmentCallSiteAttr( |
116 | callsite, | |
117 | idx.as_uint(), | |
118 | align.bytes() as u32, | |
119 | ); | |
476ff2be | 120 | } |
ff7c6d11 | 121 | regular.for_each_kind(|attr| attr.apply_callsite(idx, callsite)); |
cdc7bbd5 XL |
122 | if regular.contains(ArgAttribute::NoAliasMutRef) && should_use_mutable_noalias(cx) { |
123 | llvm::Attribute::NoAlias.apply_callsite(idx, callsite); | |
124 | } | |
fc512014 XL |
125 | match self.arg_ext { |
126 | ArgExtension::None => {} | |
127 | ArgExtension::Zext => { | |
128 | llvm::Attribute::ZExt.apply_callsite(idx, callsite); | |
129 | } | |
130 | ArgExtension::Sext => { | |
131 | llvm::Attribute::SExt.apply_callsite(idx, callsite); | |
132 | } | |
133 | } | |
476ff2be SL |
134 | } |
135 | } | |
136 | } | |
cc61c64b | 137 | |
83c7162d | 138 | pub trait LlvmType { |
b7449926 | 139 | fn llvm_type(&self, cx: &CodegenCx<'ll, '_>) -> &'ll Type; |
cc61c64b XL |
140 | } |
141 | ||
83c7162d | 142 | impl LlvmType for Reg { |
b7449926 | 143 | fn llvm_type(&self, cx: &CodegenCx<'ll, '_>) -> &'ll Type { |
cc61c64b | 144 | match self.kind { |
a1dfa0c6 | 145 | RegKind::Integer => cx.type_ix(self.size.bits()), |
dfeec247 XL |
146 | RegKind::Float => match self.size.bits() { |
147 | 32 => cx.type_f32(), | |
148 | 64 => cx.type_f64(), | |
149 | _ => bug!("unsupported float: {:?}", self), | |
150 | }, | |
151 | RegKind::Vector => cx.type_vector(cx.type_i8(), self.size.bytes()), | |
cc61c64b XL |
152 | } |
153 | } | |
154 | } | |
155 | ||
83c7162d | 156 | impl LlvmType for CastTarget { |
b7449926 | 157 | fn llvm_type(&self, cx: &CodegenCx<'ll, '_>) -> &'ll Type { |
0531ce1d | 158 | let rest_ll_unit = self.rest.unit.llvm_type(cx); |
94b46f34 XL |
159 | let (rest_count, rem_bytes) = if self.rest.unit.size.bytes() == 0 { |
160 | (0, 0) | |
161 | } else { | |
dfeec247 XL |
162 | ( |
163 | self.rest.total.bytes() / self.rest.unit.size.bytes(), | |
164 | self.rest.total.bytes() % self.rest.unit.size.bytes(), | |
165 | ) | |
94b46f34 | 166 | }; |
0531ce1d XL |
167 | |
168 | if self.prefix.iter().all(|x| x.is_none()) { | |
169 | // Simplify to a single unit when there is no prefix and size <= unit size | |
170 | if self.rest.total <= self.rest.unit.size { | |
171 | return rest_ll_unit; | |
172 | } | |
173 | ||
174 | // Simplify to array when all chunks are the same size and type | |
175 | if rem_bytes == 0 { | |
a1dfa0c6 | 176 | return cx.type_array(rest_ll_unit, rest_count); |
cc61c64b XL |
177 | } |
178 | } | |
0531ce1d XL |
179 | |
180 | // Create list of fields in the main structure | |
dfeec247 XL |
181 | let mut args: Vec<_> = self |
182 | .prefix | |
183 | .iter() | |
184 | .flat_map(|option_kind| { | |
fc512014 | 185 | option_kind.map(|kind| Reg { kind, size: self.prefix_chunk_size }.llvm_type(cx)) |
dfeec247 | 186 | }) |
0531ce1d XL |
187 | .chain((0..rest_count).map(|_| rest_ll_unit)) |
188 | .collect(); | |
189 | ||
190 | // Append final integer | |
191 | if rem_bytes != 0 { | |
192 | // Only integers can be really split further. | |
193 | assert_eq!(self.rest.unit.kind, RegKind::Integer); | |
a1dfa0c6 | 194 | args.push(cx.type_ix(rem_bytes * 8)); |
0531ce1d XL |
195 | } |
196 | ||
a1dfa0c6 | 197 | cx.type_struct(&args, false) |
cc61c64b XL |
198 | } |
199 | } | |
476ff2be | 200 | |
60c5eb7d | 201 | pub trait ArgAbiExt<'ll, 'tcx> { |
b7449926 | 202 | fn memory_ty(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type; |
a1dfa0c6 XL |
203 | fn store( |
204 | &self, | |
205 | bx: &mut Builder<'_, 'll, 'tcx>, | |
206 | val: &'ll Value, | |
207 | dst: PlaceRef<'tcx, &'ll Value>, | |
208 | ); | |
209 | fn store_fn_arg( | |
210 | &self, | |
211 | bx: &mut Builder<'_, 'll, 'tcx>, | |
212 | idx: &mut usize, | |
213 | dst: PlaceRef<'tcx, &'ll Value>, | |
214 | ); | |
54a0048b SL |
215 | } |
216 | ||
60c5eb7d | 217 | impl ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> { |
9fa01778 | 218 | /// Gets the LLVM type for a place of the original Rust type of |
0731742a | 219 | /// this argument/return, i.e., the result of `type_of::type_of`. |
b7449926 | 220 | fn memory_ty(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type { |
2c00a5a8 | 221 | self.layout.llvm_type(cx) |
54a0048b SL |
222 | } |
223 | ||
60c5eb7d | 224 | /// Stores a direct/indirect value described by this ArgAbi into a |
ff7c6d11 | 225 | /// place for the original Rust type of this argument/return. |
54a0048b SL |
226 | /// Can be used for both storing formal arguments into Rust variables |
227 | /// or results of call/invoke instructions into their destinations. | |
a1dfa0c6 XL |
228 | fn store( |
229 | &self, | |
230 | bx: &mut Builder<'_, 'll, 'tcx>, | |
231 | val: &'ll Value, | |
232 | dst: PlaceRef<'tcx, &'ll Value>, | |
233 | ) { | |
54a0048b SL |
234 | if self.is_ignore() { |
235 | return; | |
236 | } | |
b7449926 | 237 | if self.is_sized_indirect() { |
a1dfa0c6 | 238 | OperandValue::Ref(val, None, self.layout.align.abi).store(bx, dst) |
b7449926 | 239 | } else if self.is_unsized_indirect() { |
60c5eb7d | 240 | bug!("unsized `ArgAbi` must be handled through `store_fn_arg`"); |
ff7c6d11 | 241 | } else if let PassMode::Cast(cast) = self.mode { |
a7813a04 XL |
242 | // FIXME(eddyb): Figure out when the simpler Store is safe, clang |
243 | // uses it for i16 -> {i8, i8}, but not for i24 -> {i8, i8, i8}. | |
244 | let can_store_through_cast_ptr = false; | |
245 | if can_store_through_cast_ptr { | |
a1dfa0c6 XL |
246 | let cast_ptr_llty = bx.type_ptr_to(cast.llvm_type(bx)); |
247 | let cast_dst = bx.pointercast(dst.llval, cast_ptr_llty); | |
248 | bx.store(val, cast_dst, self.layout.align.abi); | |
a7813a04 XL |
249 | } else { |
250 | // The actual return type is a struct, but the ABI | |
251 | // adaptation code has cast it into some scalar type. The | |
252 | // code that follows is the only reliable way I have | |
253 | // found to do a transform like i64 -> {i32,i32}. | |
254 | // Basically we dump the data onto the stack then memcpy it. | |
255 | // | |
256 | // Other approaches I tried: | |
257 | // - Casting rust ret pointer to the foreign type and using Store | |
258 | // is (a) unsafe if size of foreign type > size of rust type and | |
259 | // (b) runs afoul of strict aliasing rules, yielding invalid | |
260 | // assembly under -O (specifically, the store gets removed). | |
261 | // - Truncating foreign type to correct integral type and then | |
262 | // bitcasting to the struct type yields invalid cast errors. | |
263 | ||
264 | // We instead thus allocate some scratch space... | |
a1dfa0c6 XL |
265 | let scratch_size = cast.size(bx); |
266 | let scratch_align = cast.align(bx); | |
e1599b0c | 267 | let llscratch = bx.alloca(cast.llvm_type(bx), scratch_align); |
2c00a5a8 | 268 | bx.lifetime_start(llscratch, scratch_size); |
a7813a04 | 269 | |
60c5eb7d | 270 | // ... where we first store the value... |
2c00a5a8 | 271 | bx.store(val, llscratch, scratch_align); |
a7813a04 | 272 | |
60c5eb7d | 273 | // ... and then memcpy it to the intended destination. |
a1dfa0c6 XL |
274 | bx.memcpy( |
275 | dst.llval, | |
276 | self.layout.align.abi, | |
277 | llscratch, | |
278 | scratch_align, | |
279 | bx.const_usize(self.layout.size.bytes()), | |
dfeec247 | 280 | MemFlags::empty(), |
a1dfa0c6 | 281 | ); |
a7813a04 | 282 | |
2c00a5a8 | 283 | bx.lifetime_end(llscratch, scratch_size); |
54a0048b SL |
284 | } |
285 | } else { | |
2c00a5a8 | 286 | OperandValue::Immediate(val).store(bx, dst); |
54a0048b SL |
287 | } |
288 | } | |
289 | ||
a1dfa0c6 XL |
290 | fn store_fn_arg( |
291 | &self, | |
292 | bx: &mut Builder<'a, 'll, 'tcx>, | |
293 | idx: &mut usize, | |
294 | dst: PlaceRef<'tcx, &'ll Value>, | |
295 | ) { | |
ff7c6d11 | 296 | let mut next = || { |
2c00a5a8 | 297 | let val = llvm::get_param(bx.llfn(), *idx as c_uint); |
54a0048b | 298 | *idx += 1; |
ff7c6d11 XL |
299 | val |
300 | }; | |
301 | match self.mode { | |
e74abb32 | 302 | PassMode::Ignore => {} |
ff7c6d11 | 303 | PassMode::Pair(..) => { |
2c00a5a8 | 304 | OperandValue::Pair(next(), next()).store(bx, dst); |
ff7c6d11 | 305 | } |
fc512014 | 306 | PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => { |
a1dfa0c6 | 307 | OperandValue::Ref(next(), Some(next()), self.layout.align.abi).store(bx, dst); |
b7449926 | 308 | } |
fc512014 XL |
309 | PassMode::Direct(_) |
310 | | PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } | |
311 | | PassMode::Cast(_) => { | |
532ac7d7 XL |
312 | let next_arg = next(); |
313 | self.store(bx, next_arg, dst); | |
ff7c6d11 | 314 | } |
54a0048b | 315 | } |
54a0048b SL |
316 | } |
317 | } | |
318 | ||
60c5eb7d | 319 | impl ArgAbiMethods<'tcx> for Builder<'a, 'll, 'tcx> { |
a1dfa0c6 XL |
320 | fn store_fn_arg( |
321 | &mut self, | |
60c5eb7d | 322 | arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, |
dfeec247 XL |
323 | idx: &mut usize, |
324 | dst: PlaceRef<'tcx, Self::Value>, | |
a1dfa0c6 | 325 | ) { |
60c5eb7d | 326 | arg_abi.store_fn_arg(self, idx, dst) |
a1dfa0c6 | 327 | } |
60c5eb7d | 328 | fn store_arg( |
a1dfa0c6 | 329 | &mut self, |
60c5eb7d | 330 | arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, |
a1dfa0c6 | 331 | val: &'ll Value, |
dfeec247 | 332 | dst: PlaceRef<'tcx, &'ll Value>, |
a1dfa0c6 | 333 | ) { |
60c5eb7d | 334 | arg_abi.store(self, val, dst) |
a1dfa0c6 | 335 | } |
60c5eb7d XL |
336 | fn arg_memory_ty(&self, arg_abi: &ArgAbi<'tcx, Ty<'tcx>>) -> &'ll Type { |
337 | arg_abi.memory_ty(self) | |
a1dfa0c6 XL |
338 | } |
339 | } | |
340 | ||
60c5eb7d | 341 | pub trait FnAbiLlvmExt<'tcx> { |
b7449926 | 342 | fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type; |
a1dfa0c6 | 343 | fn ptr_to_llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type; |
83c7162d | 344 | fn llvm_cconv(&self) -> llvm::CallConv; |
416331ca | 345 | fn apply_attrs_llfn(&self, cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value); |
a1dfa0c6 | 346 | fn apply_attrs_callsite(&self, bx: &mut Builder<'a, 'll, 'tcx>, callsite: &'ll Value); |
54a0048b SL |
347 | } |
348 | ||
60c5eb7d | 349 | impl<'tcx> FnAbiLlvmExt<'tcx> for FnAbi<'tcx, Ty<'tcx>> { |
b7449926 | 350 | fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type { |
94222f64 XL |
351 | // Ignore "extra" args from the call site for C variadic functions. |
352 | // Only the "fixed" args are part of the LLVM function signature. | |
353 | let args = if self.c_variadic { &self.args[..self.fixed_count] } else { &self.args }; | |
354 | ||
355 | let args_capacity: usize = args.iter().map(|arg| | |
8faf50e0 XL |
356 | if arg.pad.is_some() { 1 } else { 0 } + |
357 | if let PassMode::Pair(_, _) = arg.mode { 2 } else { 1 } | |
358 | ).sum(); | |
359 | let mut llargument_tys = Vec::with_capacity( | |
fc512014 | 360 | if let PassMode::Indirect { .. } = self.ret.mode { 1 } else { 0 } + args_capacity, |
8faf50e0 | 361 | ); |
54a0048b | 362 | |
ff7c6d11 | 363 | let llreturn_ty = match self.ret.mode { |
e74abb32 | 364 | PassMode::Ignore => cx.type_void(), |
dfeec247 | 365 | PassMode::Direct(_) | PassMode::Pair(..) => self.ret.layout.immediate_llvm_type(cx), |
2c00a5a8 | 366 | PassMode::Cast(cast) => cast.llvm_type(cx), |
fc512014 | 367 | PassMode::Indirect { .. } => { |
a1dfa0c6 XL |
368 | llargument_tys.push(cx.type_ptr_to(self.ret.memory_ty(cx))); |
369 | cx.type_void() | |
ff7c6d11 | 370 | } |
54a0048b SL |
371 | }; |
372 | ||
94222f64 | 373 | for arg in args { |
54a0048b SL |
374 | // add padding |
375 | if let Some(ty) = arg.pad { | |
2c00a5a8 | 376 | llargument_tys.push(ty.llvm_type(cx)); |
54a0048b SL |
377 | } |
378 | ||
ff7c6d11 | 379 | let llarg_ty = match arg.mode { |
e74abb32 | 380 | PassMode::Ignore => continue, |
2c00a5a8 | 381 | PassMode::Direct(_) => arg.layout.immediate_llvm_type(cx), |
ff7c6d11 | 382 | PassMode::Pair(..) => { |
8faf50e0 XL |
383 | llargument_tys.push(arg.layout.scalar_pair_element_llvm_type(cx, 0, true)); |
384 | llargument_tys.push(arg.layout.scalar_pair_element_llvm_type(cx, 1, true)); | |
ff7c6d11 XL |
385 | continue; |
386 | } | |
fc512014 | 387 | PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => { |
b7449926 XL |
388 | let ptr_ty = cx.tcx.mk_mut_ptr(arg.layout.ty); |
389 | let ptr_layout = cx.layout_of(ptr_ty); | |
390 | llargument_tys.push(ptr_layout.scalar_pair_element_llvm_type(cx, 0, true)); | |
391 | llargument_tys.push(ptr_layout.scalar_pair_element_llvm_type(cx, 1, true)); | |
392 | continue; | |
393 | } | |
2c00a5a8 | 394 | PassMode::Cast(cast) => cast.llvm_type(cx), |
fc512014 XL |
395 | PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } => { |
396 | cx.type_ptr_to(arg.memory_ty(cx)) | |
397 | } | |
54a0048b | 398 | }; |
54a0048b SL |
399 | llargument_tys.push(llarg_ty); |
400 | } | |
401 | ||
532ac7d7 | 402 | if self.c_variadic { |
a1dfa0c6 | 403 | cx.type_variadic_func(&llargument_tys, llreturn_ty) |
54a0048b | 404 | } else { |
a1dfa0c6 XL |
405 | cx.type_func(&llargument_tys, llreturn_ty) |
406 | } | |
407 | } | |
408 | ||
409 | fn ptr_to_llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type { | |
410 | unsafe { | |
dfeec247 XL |
411 | llvm::LLVMPointerType( |
412 | self.llvm_type(cx), | |
3dfed10e | 413 | cx.data_layout().instruction_address_space.0 as c_uint, |
dfeec247 | 414 | ) |
54a0048b SL |
415 | } |
416 | } | |
417 | ||
83c7162d XL |
418 | fn llvm_cconv(&self) -> llvm::CallConv { |
419 | match self.conv { | |
5869c6ff | 420 | Conv::C | Conv::Rust | Conv::CCmseNonSecureCall => llvm::CCallConv, |
8faf50e0 | 421 | Conv::AmdGpuKernel => llvm::AmdGpuKernel, |
f035d41b XL |
422 | Conv::AvrInterrupt => llvm::AvrInterrupt, |
423 | Conv::AvrNonBlockingInterrupt => llvm::AvrNonBlockingInterrupt, | |
83c7162d XL |
424 | Conv::ArmAapcs => llvm::ArmAapcsCallConv, |
425 | Conv::Msp430Intr => llvm::Msp430Intr, | |
426 | Conv::PtxKernel => llvm::PtxKernel, | |
427 | Conv::X86Fastcall => llvm::X86FastcallCallConv, | |
428 | Conv::X86Intr => llvm::X86_Intr, | |
429 | Conv::X86Stdcall => llvm::X86StdcallCallConv, | |
430 | Conv::X86ThisCall => llvm::X86_ThisCall, | |
431 | Conv::X86VectorCall => llvm::X86_VectorCall, | |
432 | Conv::X86_64SysV => llvm::X86_64_SysV, | |
433 | Conv::X86_64Win64 => llvm::X86_64_Win64, | |
434 | } | |
435 | } | |
436 | ||
416331ca | 437 | fn apply_attrs_llfn(&self, cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value) { |
60c5eb7d XL |
438 | // FIXME(eddyb) can this also be applied to callsites? |
439 | if self.ret.layout.abi.is_uninhabited() { | |
440 | llvm::Attribute::NoReturn.apply_llfn(llvm::AttributePlace::Function, llfn); | |
441 | } | |
442 | ||
ba9703b0 XL |
443 | // FIXME(eddyb, wesleywiser): apply this to callsites as well? |
444 | if !self.can_unwind { | |
445 | llvm::Attribute::NoUnwind.apply_llfn(llvm::AttributePlace::Function, llfn); | |
446 | } | |
447 | ||
ff7c6d11 | 448 | let mut i = 0; |
fc512014 | 449 | let mut apply = |attrs: &ArgAttributes| { |
cdc7bbd5 | 450 | attrs.apply_attrs_to_llfn(llvm::AttributePlace::Argument(i), cx, llfn); |
ff7c6d11 | 451 | i += 1; |
fc512014 | 452 | i - 1 |
ff7c6d11 XL |
453 | }; |
454 | match self.ret.mode { | |
455 | PassMode::Direct(ref attrs) => { | |
cdc7bbd5 | 456 | attrs.apply_attrs_to_llfn(llvm::AttributePlace::ReturnValue, cx, llfn); |
fc512014 XL |
457 | } |
458 | PassMode::Indirect { ref attrs, extra_attrs: _, on_stack } => { | |
459 | assert!(!on_stack); | |
460 | let i = apply(attrs); | |
6a06907d XL |
461 | unsafe { |
462 | llvm::LLVMRustAddStructRetAttr( | |
463 | llfn, | |
464 | llvm::AttributePlace::Argument(i).as_uint(), | |
465 | self.ret.layout.llvm_type(cx), | |
466 | ); | |
467 | } | |
ff7c6d11 | 468 | } |
ff7c6d11 | 469 | _ => {} |
54a0048b | 470 | } |
54a0048b | 471 | for arg in &self.args { |
ff7c6d11 | 472 | if arg.pad.is_some() { |
fc512014 | 473 | apply(&ArgAttributes::new()); |
ff7c6d11 XL |
474 | } |
475 | match arg.mode { | |
e74abb32 | 476 | PassMode::Ignore => {} |
fc512014 XL |
477 | PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: true } => { |
478 | let i = apply(attrs); | |
479 | unsafe { | |
480 | llvm::LLVMRustAddByValAttr( | |
481 | llfn, | |
482 | llvm::AttributePlace::Argument(i).as_uint(), | |
483 | arg.layout.llvm_type(cx), | |
484 | ); | |
485 | } | |
dfeec247 | 486 | } |
fc512014 XL |
487 | PassMode::Direct(ref attrs) |
488 | | PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: false } => { | |
489 | apply(attrs); | |
490 | } | |
491 | PassMode::Indirect { ref attrs, extra_attrs: Some(ref extra_attrs), on_stack } => { | |
492 | assert!(!on_stack); | |
493 | apply(attrs); | |
494 | apply(extra_attrs); | |
b7449926 | 495 | } |
ff7c6d11 | 496 | PassMode::Pair(ref a, ref b) => { |
fc512014 XL |
497 | apply(a); |
498 | apply(b); | |
499 | } | |
500 | PassMode::Cast(_) => { | |
501 | apply(&ArgAttributes::new()); | |
ff7c6d11 | 502 | } |
54a0048b SL |
503 | } |
504 | } | |
505 | } | |
506 | ||
a1dfa0c6 | 507 | fn apply_attrs_callsite(&self, bx: &mut Builder<'a, 'll, 'tcx>, callsite: &'ll Value) { |
c295e0f8 XL |
508 | if self.ret.layout.abi.is_uninhabited() { |
509 | llvm::Attribute::NoReturn.apply_callsite(llvm::AttributePlace::Function, callsite); | |
510 | } | |
511 | if !self.can_unwind { | |
512 | llvm::Attribute::NoUnwind.apply_callsite(llvm::AttributePlace::Function, callsite); | |
513 | } | |
ba9703b0 | 514 | |
ff7c6d11 | 515 | let mut i = 0; |
cdc7bbd5 XL |
516 | let mut apply = |cx: &CodegenCx<'_, '_>, attrs: &ArgAttributes| { |
517 | attrs.apply_attrs_to_callsite(llvm::AttributePlace::Argument(i), cx, callsite); | |
ff7c6d11 | 518 | i += 1; |
fc512014 | 519 | i - 1 |
ff7c6d11 XL |
520 | }; |
521 | match self.ret.mode { | |
522 | PassMode::Direct(ref attrs) => { | |
c295e0f8 | 523 | attrs.apply_attrs_to_callsite(llvm::AttributePlace::ReturnValue, bx.cx, callsite); |
fc512014 XL |
524 | } |
525 | PassMode::Indirect { ref attrs, extra_attrs: _, on_stack } => { | |
526 | assert!(!on_stack); | |
cdc7bbd5 | 527 | let i = apply(bx.cx, attrs); |
6a06907d XL |
528 | unsafe { |
529 | llvm::LLVMRustAddStructRetCallSiteAttr( | |
530 | callsite, | |
531 | llvm::AttributePlace::Argument(i).as_uint(), | |
532 | self.ret.layout.llvm_type(bx), | |
533 | ); | |
534 | } | |
ff7c6d11 | 535 | } |
ff7c6d11 | 536 | _ => {} |
54a0048b | 537 | } |
c295e0f8 | 538 | if let abi::Abi::Scalar(scalar) = self.ret.layout.abi { |
83c7162d XL |
539 | // If the value is a boolean, the range is 0..2 and that ultimately |
540 | // become 0..0 when the type becomes i1, which would be rejected | |
541 | // by the LLVM verifier. | |
ba9703b0 | 542 | if let Int(..) = scalar.value { |
c295e0f8 XL |
543 | if !scalar.is_bool() && !scalar.is_always_valid(bx) { |
544 | bx.range_metadata(callsite, scalar.valid_range); | |
83c7162d | 545 | } |
83c7162d XL |
546 | } |
547 | } | |
54a0048b | 548 | for arg in &self.args { |
ff7c6d11 | 549 | if arg.pad.is_some() { |
cdc7bbd5 | 550 | apply(bx.cx, &ArgAttributes::new()); |
ff7c6d11 XL |
551 | } |
552 | match arg.mode { | |
e74abb32 | 553 | PassMode::Ignore => {} |
fc512014 | 554 | PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: true } => { |
cdc7bbd5 | 555 | let i = apply(bx.cx, attrs); |
fc512014 XL |
556 | unsafe { |
557 | llvm::LLVMRustAddByValCallSiteAttr( | |
558 | callsite, | |
559 | llvm::AttributePlace::Argument(i).as_uint(), | |
560 | arg.layout.llvm_type(bx), | |
561 | ); | |
562 | } | |
dfeec247 | 563 | } |
fc512014 XL |
564 | PassMode::Direct(ref attrs) |
565 | | PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: false } => { | |
cdc7bbd5 | 566 | apply(bx.cx, attrs); |
fc512014 XL |
567 | } |
568 | PassMode::Indirect { | |
569 | ref attrs, | |
570 | extra_attrs: Some(ref extra_attrs), | |
571 | on_stack: _, | |
572 | } => { | |
cdc7bbd5 XL |
573 | apply(bx.cx, attrs); |
574 | apply(bx.cx, extra_attrs); | |
b7449926 | 575 | } |
ff7c6d11 | 576 | PassMode::Pair(ref a, ref b) => { |
cdc7bbd5 XL |
577 | apply(bx.cx, a); |
578 | apply(bx.cx, b); | |
fc512014 XL |
579 | } |
580 | PassMode::Cast(_) => { | |
cdc7bbd5 | 581 | apply(bx.cx, &ArgAttributes::new()); |
ff7c6d11 | 582 | } |
54a0048b SL |
583 | } |
584 | } | |
585 | ||
83c7162d XL |
586 | let cconv = self.llvm_cconv(); |
587 | if cconv != llvm::CCallConv { | |
588 | llvm::SetInstructionCallConv(callsite, cconv); | |
54a0048b | 589 | } |
5869c6ff XL |
590 | |
591 | if self.conv == Conv::CCmseNonSecureCall { | |
592 | // This will probably get ignored on all targets but those supporting the TrustZone-M | |
593 | // extension (thumbv8m targets). | |
594 | unsafe { | |
595 | llvm::AddCallSiteAttrString( | |
596 | callsite, | |
597 | llvm::AttributePlace::Function, | |
6a06907d | 598 | cstr::cstr!("cmse_nonsecure_call"), |
5869c6ff XL |
599 | ); |
600 | } | |
601 | } | |
54a0048b SL |
602 | } |
603 | } | |
a1dfa0c6 | 604 | |
a1dfa0c6 | 605 | impl AbiBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> { |
dfeec247 | 606 | fn apply_attrs_callsite(&mut self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>, callsite: Self::Value) { |
60c5eb7d | 607 | fn_abi.apply_attrs_callsite(self, callsite) |
a1dfa0c6 | 608 | } |
532ac7d7 | 609 | |
3c0e092e | 610 | fn get_param(&mut self, index: usize) -> Self::Value { |
532ac7d7 XL |
611 | llvm::get_param(self.llfn(), index as c_uint) |
612 | } | |
a1dfa0c6 | 613 | } |