]>
Commit | Line | Data |
---|---|---|
60c5eb7d XL |
1 | use super::operand::{OperandRef, OperandValue}; |
2 | use super::place::PlaceRef; | |
dfeec247 | 3 | use super::{FunctionCx, LocalRef}; |
60c5eb7d XL |
4 | |
5 | use crate::base; | |
5099ac24 | 6 | use crate::common::{self, IntPredicate}; |
f2b60f7d | 7 | use crate::meth::get_vtable; |
60c5eb7d | 8 | use crate::traits::*; |
dfeec247 | 9 | use crate::MemFlags; |
60c5eb7d | 10 | |
ba9703b0 | 11 | use rustc_middle::mir; |
064997fb | 12 | use rustc_middle::mir::Operand; |
ba9703b0 | 13 | use rustc_middle::ty::cast::{CastTy, IntTy}; |
c295e0f8 | 14 | use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf}; |
ba9703b0 | 15 | use rustc_middle::ty::{self, adjustment::PointerCast, Instance, Ty, TyCtxt}; |
dfeec247 | 16 | use rustc_span::source_map::{Span, DUMMY_SP}; |
92a42be0 | 17 | |
dc9dc135 | 18 | impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { |
064997fb | 19 | #[instrument(level = "trace", skip(self, bx))] |
a1dfa0c6 XL |
20 | pub fn codegen_rvalue( |
21 | &mut self, | |
22 | mut bx: Bx, | |
23 | dest: PlaceRef<'tcx, Bx::Value>, | |
dfeec247 | 24 | rvalue: &mir::Rvalue<'tcx>, |
a1dfa0c6 | 25 | ) -> Bx { |
92a42be0 | 26 | match *rvalue { |
dfeec247 XL |
27 | mir::Rvalue::Use(ref operand) => { |
28 | let cg_operand = self.codegen_operand(&mut bx, operand); | |
29 | // FIXME: consider not copying constants through stack. (Fixable by codegen'ing | |
30 | // constants into `OperandValue::Ref`; why don’t we do that yet if we don’t?) | |
31 | cg_operand.val.store(&mut bx, dest); | |
32 | bx | |
33 | } | |
92a42be0 | 34 | |
48663c56 | 35 | mir::Rvalue::Cast(mir::CastKind::Pointer(PointerCast::Unsize), ref source, _) => { |
ff7c6d11 XL |
36 | // The destination necessarily contains a fat pointer, so if |
37 | // it's a scalar pair, it's a fat pointer or newtype thereof. | |
a1dfa0c6 | 38 | if bx.cx().is_backend_scalar_pair(dest.layout) { |
60c5eb7d | 39 | // Into-coerce of a thin pointer to a fat pointer -- just |
92a42be0 | 40 | // use the operand path. |
a1dfa0c6 XL |
41 | let (mut bx, temp) = self.codegen_rvalue_operand(bx, rvalue); |
42 | temp.val.store(&mut bx, dest); | |
2c00a5a8 | 43 | return bx; |
92a42be0 SL |
44 | } |
45 | ||
46 | // Unsize of a nontrivial struct. I would prefer for | |
94b46f34 | 47 | // this to be eliminated by MIR building, but |
92a42be0 SL |
48 | // `CoerceUnsized` can be passed by a where-clause, |
49 | // so the (generic) MIR may not be able to expand it. | |
a1dfa0c6 | 50 | let operand = self.codegen_operand(&mut bx, source); |
ff7c6d11 | 51 | match operand.val { |
dfeec247 | 52 | OperandValue::Pair(..) | OperandValue::Immediate(_) => { |
60c5eb7d | 53 | // Unsize from an immediate structure. We don't |
32a655c1 SL |
54 | // really need a temporary alloca here, but |
55 | // avoiding it would require us to have | |
60c5eb7d | 56 | // `coerce_unsized_into` use `extractvalue` to |
32a655c1 SL |
57 | // index into the struct, and this case isn't |
58 | // important enough for it. | |
94b46f34 | 59 | debug!("codegen_rvalue: creating ugly alloca"); |
e1599b0c | 60 | let scratch = PlaceRef::alloca(&mut bx, operand.layout); |
a1dfa0c6 XL |
61 | scratch.storage_live(&mut bx); |
62 | operand.val.store(&mut bx, scratch); | |
63 | base::coerce_unsized_into(&mut bx, scratch, dest); | |
64 | scratch.storage_dead(&mut bx); | |
32a655c1 | 65 | } |
b7449926 | 66 | OperandValue::Ref(llref, None, align) => { |
e1599b0c | 67 | let source = PlaceRef::new_sized_aligned(llref, operand.layout, align); |
a1dfa0c6 | 68 | base::coerce_unsized_into(&mut bx, source, dest); |
92a42be0 | 69 | } |
b7449926 | 70 | OperandValue::Ref(_, Some(_), _) => { |
60c5eb7d | 71 | bug!("unsized coercion on an unsized rvalue"); |
b7449926 | 72 | } |
ff7c6d11 | 73 | } |
2c00a5a8 | 74 | bx |
92a42be0 SL |
75 | } |
76 | ||
ea8adc8c | 77 | mir::Rvalue::Repeat(ref elem, count) => { |
a1dfa0c6 | 78 | let cg_elem = self.codegen_operand(&mut bx, elem); |
3b2f2976 | 79 | |
ff7c6d11 XL |
80 | // Do not generate the loop for zero-sized elements or empty arrays. |
81 | if dest.layout.is_zst() { | |
2c00a5a8 | 82 | return bx; |
3b2f2976 | 83 | } |
3b2f2976 | 84 | |
94b46f34 | 85 | if let OperandValue::Immediate(v) = cg_elem.val { |
532ac7d7 XL |
86 | let zero = bx.const_usize(0); |
87 | let start = dest.project_index(&mut bx, zero).llval; | |
88 | let size = bx.const_usize(dest.layout.size.bytes()); | |
ff7c6d11 | 89 | |
3b2f2976 | 90 | // Use llvm.memset.p0i8.* to initialize all zero arrays |
f2b60f7d | 91 | if bx.cx().const_to_opt_u128(v, false) == Some(0) { |
a1dfa0c6 XL |
92 | let fill = bx.cx().const_u8(0); |
93 | bx.memset(start, fill, size, dest.align, MemFlags::empty()); | |
2c00a5a8 | 94 | return bx; |
3b2f2976 XL |
95 | } |
96 | ||
97 | // Use llvm.memset.p0i8.* to initialize byte arrays | |
1b1a35ee | 98 | let v = bx.from_immediate(v); |
a1dfa0c6 XL |
99 | if bx.cx().val_ty(v) == bx.cx().type_i8() { |
100 | bx.memset(start, v, size, dest.align, MemFlags::empty()); | |
2c00a5a8 | 101 | return bx; |
3b2f2976 XL |
102 | } |
103 | } | |
104 | ||
ba9703b0 | 105 | let count = |
fc512014 | 106 | self.monomorphize(count).eval_usize(bx.cx().tcx(), ty::ParamEnv::reveal_all()); |
ba9703b0 | 107 | |
532ac7d7 | 108 | bx.write_operand_repeatedly(cg_elem, count, dest) |
92a42be0 SL |
109 | } |
110 | ||
9cc50fc6 | 111 | mir::Rvalue::Aggregate(ref kind, ref operands) => { |
ff7c6d11 | 112 | let (dest, active_field_index) = match **kind { |
a2a8927a | 113 | mir::AggregateKind::Adt(adt_did, variant_index, _, _, active_field_index) => { |
a1dfa0c6 | 114 | dest.codegen_set_discr(&mut bx, variant_index); |
a2a8927a | 115 | if bx.tcx().adt_def(adt_did).is_enum() { |
a1dfa0c6 | 116 | (dest.project_downcast(&mut bx, variant_index), active_field_index) |
ff7c6d11 XL |
117 | } else { |
118 | (dest, active_field_index) | |
9cc50fc6 SL |
119 | } |
120 | } | |
dfeec247 | 121 | _ => (dest, None), |
ff7c6d11 XL |
122 | }; |
123 | for (i, operand) in operands.iter().enumerate() { | |
a1dfa0c6 | 124 | let op = self.codegen_operand(&mut bx, operand); |
ff7c6d11 XL |
125 | // Do not generate stores and GEPis for zero-sized fields. |
126 | if !op.layout.is_zst() { | |
127 | let field_index = active_field_index.unwrap_or(i); | |
064997fb FG |
128 | let field = if let mir::AggregateKind::Array(_) = **kind { |
129 | let llindex = bx.cx().const_usize(field_index as u64); | |
130 | dest.project_index(&mut bx, llindex) | |
131 | } else { | |
132 | dest.project_field(&mut bx, field_index) | |
133 | }; | |
a1dfa0c6 | 134 | op.val.store(&mut bx, field); |
ff7c6d11 | 135 | } |
92a42be0 | 136 | } |
2c00a5a8 | 137 | bx |
92a42be0 SL |
138 | } |
139 | ||
92a42be0 | 140 | _ => { |
416331ca | 141 | assert!(self.rvalue_creates_operand(rvalue, DUMMY_SP)); |
a1dfa0c6 XL |
142 | let (mut bx, temp) = self.codegen_rvalue_operand(bx, rvalue); |
143 | temp.val.store(&mut bx, dest); | |
2c00a5a8 | 144 | bx |
92a42be0 SL |
145 | } |
146 | } | |
147 | } | |
148 | ||
a1dfa0c6 XL |
149 | pub fn codegen_rvalue_unsized( |
150 | &mut self, | |
151 | mut bx: Bx, | |
152 | indirect_dest: PlaceRef<'tcx, Bx::Value>, | |
153 | rvalue: &mir::Rvalue<'tcx>, | |
154 | ) -> Bx { | |
dfeec247 XL |
155 | debug!( |
156 | "codegen_rvalue_unsized(indirect_dest.llval={:?}, rvalue={:?})", | |
157 | indirect_dest.llval, rvalue | |
158 | ); | |
b7449926 XL |
159 | |
160 | match *rvalue { | |
161 | mir::Rvalue::Use(ref operand) => { | |
a1dfa0c6 XL |
162 | let cg_operand = self.codegen_operand(&mut bx, operand); |
163 | cg_operand.val.store_unsized(&mut bx, indirect_dest); | |
b7449926 XL |
164 | bx |
165 | } | |
166 | ||
60c5eb7d | 167 | _ => bug!("unsized assignment other than `Rvalue::Use`"), |
b7449926 XL |
168 | } |
169 | } | |
170 | ||
a1dfa0c6 XL |
171 | pub fn codegen_rvalue_operand( |
172 | &mut self, | |
173 | mut bx: Bx, | |
dfeec247 | 174 | rvalue: &mir::Rvalue<'tcx>, |
a1dfa0c6 | 175 | ) -> (Bx, OperandRef<'tcx, Bx::Value>) { |
416331ca XL |
176 | assert!( |
177 | self.rvalue_creates_operand(rvalue, DUMMY_SP), | |
178 | "cannot codegen {:?} to operand", | |
179 | rvalue, | |
180 | ); | |
92a42be0 SL |
181 | |
182 | match *rvalue { | |
ff7c6d11 | 183 | mir::Rvalue::Cast(ref kind, ref source, mir_cast_ty) => { |
a1dfa0c6 | 184 | let operand = self.codegen_operand(&mut bx, source); |
54a0048b | 185 | debug!("cast operand is {:?}", operand); |
fc512014 | 186 | let cast = bx.cx().layout_of(self.monomorphize(mir_cast_ty)); |
92a42be0 SL |
187 | |
188 | let val = match *kind { | |
923072b8 FG |
189 | mir::CastKind::PointerExposeAddress => { |
190 | assert!(bx.cx().is_backend_immediate(cast)); | |
191 | let llptr = operand.immediate(); | |
192 | let llcast_ty = bx.cx().immediate_backend_type(cast); | |
193 | let lladdr = bx.ptrtoint(llptr, llcast_ty); | |
194 | OperandValue::Immediate(lladdr) | |
195 | } | |
48663c56 | 196 | mir::CastKind::Pointer(PointerCast::ReifyFnPointer) => { |
1b1a35ee | 197 | match *operand.layout.ty.kind() { |
b7449926 | 198 | ty::FnDef(def_id, substs) => { |
3dfed10e XL |
199 | let instance = ty::Instance::resolve_for_fn_ptr( |
200 | bx.tcx(), | |
201 | ty::ParamEnv::reveal_all(), | |
202 | def_id, | |
203 | substs, | |
e74abb32 | 204 | ) |
3dfed10e XL |
205 | .unwrap() |
206 | .polymorphize(bx.cx().tcx()); | |
207 | OperandValue::Immediate(bx.get_fn_addr(instance)) | |
54a0048b | 208 | } |
dfeec247 | 209 | _ => bug!("{} cannot be reified to a fn ptr", operand.layout.ty), |
54a0048b SL |
210 | } |
211 | } | |
48663c56 | 212 | mir::CastKind::Pointer(PointerCast::ClosureFnPointer(_)) => { |
1b1a35ee | 213 | match *operand.layout.ty.kind() { |
b7449926 | 214 | ty::Closure(def_id, substs) => { |
dc9dc135 | 215 | let instance = Instance::resolve_closure( |
e74abb32 XL |
216 | bx.cx().tcx(), |
217 | def_id, | |
218 | substs, | |
dfeec247 | 219 | ty::ClosureKind::FnOnce, |
3dfed10e | 220 | ) |
064997fb | 221 | .expect("failed to normalize and resolve closure during codegen") |
3dfed10e | 222 | .polymorphize(bx.cx().tcx()); |
e74abb32 | 223 | OperandValue::Immediate(bx.cx().get_fn_addr(instance)) |
8bb4bdeb | 224 | } |
dfeec247 | 225 | _ => bug!("{} cannot be cast to a fn ptr", operand.layout.ty), |
8bb4bdeb XL |
226 | } |
227 | } | |
48663c56 | 228 | mir::CastKind::Pointer(PointerCast::UnsafeFnPointer) => { |
60c5eb7d | 229 | // This is a no-op at the LLVM level. |
92a42be0 SL |
230 | operand.val |
231 | } | |
48663c56 | 232 | mir::CastKind::Pointer(PointerCast::Unsize) => { |
a1dfa0c6 | 233 | assert!(bx.cx().is_backend_scalar_pair(cast)); |
94222f64 | 234 | let (lldata, llextra) = match operand.val { |
3157f602 | 235 | OperandValue::Pair(lldata, llextra) => { |
60c5eb7d | 236 | // unsize from a fat pointer -- this is a |
94222f64 XL |
237 | // "trait-object-to-supertrait" coercion. |
238 | (lldata, Some(llextra)) | |
92a42be0 SL |
239 | } |
240 | OperandValue::Immediate(lldata) => { | |
241 | // "standard" unsize | |
94222f64 | 242 | (lldata, None) |
92a42be0 | 243 | } |
32a655c1 | 244 | OperandValue::Ref(..) => { |
dfeec247 | 245 | bug!("by-ref operand {:?} in `codegen_rvalue_operand`", operand); |
92a42be0 | 246 | } |
94222f64 XL |
247 | }; |
248 | let (lldata, llextra) = | |
249 | base::unsize_ptr(&mut bx, lldata, operand.layout.ty, cast.ty, llextra); | |
250 | OperandValue::Pair(lldata, llextra) | |
92a42be0 | 251 | } |
dfeec247 XL |
252 | mir::CastKind::Pointer(PointerCast::MutToConstPointer) |
253 | | mir::CastKind::Misc | |
254 | if bx.cx().is_backend_scalar_pair(operand.layout) => | |
255 | { | |
ff7c6d11 | 256 | if let OperandValue::Pair(data_ptr, meta) = operand.val { |
a1dfa0c6 | 257 | if bx.cx().is_backend_scalar_pair(cast) { |
dfeec247 XL |
258 | let data_cast = bx.pointercast( |
259 | data_ptr, | |
260 | bx.cx().scalar_pair_element_backend_type(cast, 0, true), | |
261 | ); | |
ff7c6d11 | 262 | OperandValue::Pair(data_cast, meta) |
dfeec247 XL |
263 | } else { |
264 | // cast to thin-ptr | |
5bcae85e SL |
265 | // Cast of fat-ptr to thin-ptr is an extraction of data-ptr and |
266 | // pointer-cast of that pointer to desired pointer type. | |
a1dfa0c6 | 267 | let llcast_ty = bx.cx().immediate_backend_type(cast); |
2c00a5a8 | 268 | let llval = bx.pointercast(data_ptr, llcast_ty); |
5bcae85e SL |
269 | OperandValue::Immediate(llval) |
270 | } | |
271 | } else { | |
60c5eb7d | 272 | bug!("unexpected non-pair operand"); |
5bcae85e SL |
273 | } |
274 | } | |
f2b60f7d FG |
275 | mir::CastKind::DynStar => { |
276 | let data = match operand.val { | |
277 | OperandValue::Ref(_, _, _) => todo!(), | |
278 | OperandValue::Immediate(v) => v, | |
279 | OperandValue::Pair(_, _) => todo!(), | |
280 | }; | |
281 | let trait_ref = | |
282 | if let ty::Dynamic(data, _, ty::DynStar) = cast.ty.kind() { | |
283 | data.principal() | |
284 | } else { | |
285 | bug!("Only valid to do a DynStar cast into a DynStar type") | |
286 | }; | |
287 | let vtable = get_vtable(bx.cx(), source.ty(self.mir, bx.tcx()), trait_ref); | |
288 | OperandValue::Pair(data, vtable) | |
289 | } | |
ba9703b0 XL |
290 | mir::CastKind::Pointer( |
291 | PointerCast::MutToConstPointer | PointerCast::ArrayToPointer, | |
292 | ) | |
923072b8 FG |
293 | | mir::CastKind::Misc |
294 | // Since int2ptr can have arbitrary integer types as input (so we have to do | |
295 | // sign extension and all that), it is currently best handled in the same code | |
296 | // path as the other integer-to-X casts. | |
297 | | mir::CastKind::PointerFromExposedAddress => { | |
a1dfa0c6 XL |
298 | assert!(bx.cx().is_backend_immediate(cast)); |
299 | let ll_t_out = bx.cx().immediate_backend_type(cast); | |
0bf4aa26 | 300 | if operand.layout.abi.is_uninhabited() { |
a1dfa0c6 | 301 | let val = OperandValue::Immediate(bx.cx().const_undef(ll_t_out)); |
dfeec247 | 302 | return (bx, OperandRef { val, layout: cast }); |
83c7162d | 303 | } |
dfeec247 XL |
304 | let r_t_in = |
305 | CastTy::from_ty(operand.layout.ty).expect("bad input type for cast"); | |
ff7c6d11 | 306 | let r_t_out = CastTy::from_ty(cast.ty).expect("bad output type for cast"); |
a1dfa0c6 | 307 | let ll_t_in = bx.cx().immediate_backend_type(operand.layout); |
32a655c1 | 308 | let llval = operand.immediate(); |
c30ab7b3 | 309 | |
9cc50fc6 | 310 | let newval = match (r_t_in, r_t_out) { |
064997fb FG |
311 | (CastTy::Int(i), CastTy::Int(_)) => { |
312 | bx.intcast(llval, ll_t_out, i.is_signed()) | |
313 | } | |
9cc50fc6 | 314 | (CastTy::Float, CastTy::Float) => { |
a1dfa0c6 XL |
315 | let srcsz = bx.cx().float_width(ll_t_in); |
316 | let dstsz = bx.cx().float_width(ll_t_out); | |
9cc50fc6 | 317 | if dstsz > srcsz { |
2c00a5a8 | 318 | bx.fpext(llval, ll_t_out) |
9cc50fc6 | 319 | } else if srcsz > dstsz { |
2c00a5a8 | 320 | bx.fptrunc(llval, ll_t_out) |
9cc50fc6 SL |
321 | } else { |
322 | llval | |
323 | } | |
324 | } | |
064997fb FG |
325 | (CastTy::Int(i), CastTy::Float) => { |
326 | if i.is_signed() { | |
dfeec247 XL |
327 | bx.sitofp(llval, ll_t_out) |
328 | } else { | |
329 | bx.uitofp(llval, ll_t_out) | |
330 | } | |
331 | } | |
ba9703b0 | 332 | (CastTy::Ptr(_) | CastTy::FnPtr, CastTy::Ptr(_)) => { |
dfeec247 XL |
333 | bx.pointercast(llval, ll_t_out) |
334 | } | |
064997fb FG |
335 | (CastTy::Int(i), CastTy::Ptr(_)) => { |
336 | let usize_llval = | |
337 | bx.intcast(llval, bx.cx().type_isize(), i.is_signed()); | |
2c00a5a8 XL |
338 | bx.inttoptr(usize_llval, ll_t_out) |
339 | } | |
dfeec247 | 340 | (CastTy::Float, CastTy::Int(IntTy::I)) => { |
5099ac24 | 341 | bx.cast_float_to_int(true, llval, ll_t_out) |
dfeec247 XL |
342 | } |
343 | (CastTy::Float, CastTy::Int(_)) => { | |
5099ac24 | 344 | bx.cast_float_to_int(false, llval, ll_t_out) |
dfeec247 XL |
345 | } |
346 | _ => bug!("unsupported cast: {:?} to {:?}", operand.layout.ty, cast.ty), | |
9cc50fc6 SL |
347 | }; |
348 | OperandValue::Immediate(newval) | |
349 | } | |
92a42be0 | 350 | }; |
dfeec247 | 351 | (bx, OperandRef { val, layout: cast }) |
92a42be0 SL |
352 | } |
353 | ||
ba9703b0 | 354 | mir::Rvalue::Ref(_, bk, place) => { |
dfeec247 XL |
355 | let mk_ref = move |tcx: TyCtxt<'tcx>, ty: Ty<'tcx>| { |
356 | tcx.mk_ref( | |
357 | tcx.lifetimes.re_erased, | |
358 | ty::TypeAndMut { ty, mutbl: bk.to_mutbl_lossy() }, | |
359 | ) | |
360 | }; | |
361 | self.codegen_place_to_pointer(bx, place, mk_ref) | |
362 | } | |
92a42be0 | 363 | |
064997fb FG |
364 | mir::Rvalue::CopyForDeref(place) => { |
365 | let operand = self.codegen_operand(&mut bx, &Operand::Copy(place)); | |
366 | (bx, operand) | |
367 | } | |
ba9703b0 | 368 | mir::Rvalue::AddressOf(mutability, place) => { |
dfeec247 | 369 | let mk_ptr = move |tcx: TyCtxt<'tcx>, ty: Ty<'tcx>| { |
74b04a01 | 370 | tcx.mk_ptr(ty::TypeAndMut { ty, mutbl: mutability }) |
7453a54e | 371 | }; |
dfeec247 | 372 | self.codegen_place_to_pointer(bx, place, mk_ptr) |
92a42be0 SL |
373 | } |
374 | ||
ba9703b0 | 375 | mir::Rvalue::Len(place) => { |
a1dfa0c6 | 376 | let size = self.evaluate_array_len(&mut bx, place); |
7453a54e | 377 | let operand = OperandRef { |
3b2f2976 | 378 | val: OperandValue::Immediate(size), |
a1dfa0c6 | 379 | layout: bx.cx().layout_of(bx.tcx().types.usize), |
7453a54e | 380 | }; |
2c00a5a8 | 381 | (bx, operand) |
92a42be0 SL |
382 | } |
383 | ||
6a06907d | 384 | mir::Rvalue::BinaryOp(op, box (ref lhs, ref rhs)) => { |
a1dfa0c6 XL |
385 | let lhs = self.codegen_operand(&mut bx, lhs); |
386 | let rhs = self.codegen_operand(&mut bx, rhs); | |
ff7c6d11 | 387 | let llresult = match (lhs.val, rhs.val) { |
dfeec247 XL |
388 | ( |
389 | OperandValue::Pair(lhs_addr, lhs_extra), | |
390 | OperandValue::Pair(rhs_addr, rhs_extra), | |
391 | ) => self.codegen_fat_ptr_binop( | |
392 | &mut bx, | |
393 | op, | |
394 | lhs_addr, | |
395 | lhs_extra, | |
396 | rhs_addr, | |
397 | rhs_extra, | |
398 | lhs.layout.ty, | |
399 | ), | |
400 | ||
401 | (OperandValue::Immediate(lhs_val), OperandValue::Immediate(rhs_val)) => { | |
a1dfa0c6 | 402 | self.codegen_scalar_binop(&mut bx, op, lhs_val, rhs_val, lhs.layout.ty) |
ff7c6d11 XL |
403 | } |
404 | ||
dfeec247 | 405 | _ => bug!(), |
92a42be0 | 406 | }; |
7453a54e | 407 | let operand = OperandRef { |
92a42be0 | 408 | val: OperandValue::Immediate(llresult), |
dfeec247 | 409 | layout: bx.cx().layout_of(op.ty(bx.tcx(), lhs.layout.ty, rhs.layout.ty)), |
7453a54e | 410 | }; |
2c00a5a8 | 411 | (bx, operand) |
92a42be0 | 412 | } |
6a06907d | 413 | mir::Rvalue::CheckedBinaryOp(op, box (ref lhs, ref rhs)) => { |
a1dfa0c6 XL |
414 | let lhs = self.codegen_operand(&mut bx, lhs); |
415 | let rhs = self.codegen_operand(&mut bx, rhs); | |
dfeec247 XL |
416 | let result = self.codegen_scalar_checked_binop( |
417 | &mut bx, | |
418 | op, | |
419 | lhs.immediate(), | |
420 | rhs.immediate(), | |
421 | lhs.layout.ty, | |
422 | ); | |
2c00a5a8 | 423 | let val_ty = op.ty(bx.tcx(), lhs.layout.ty, rhs.layout.ty); |
0531ce1d | 424 | let operand_ty = bx.tcx().intern_tup(&[val_ty, bx.tcx().types.bool]); |
dfeec247 | 425 | let operand = OperandRef { val: result, layout: bx.cx().layout_of(operand_ty) }; |
3157f602 | 426 | |
2c00a5a8 | 427 | (bx, operand) |
3157f602 | 428 | } |
92a42be0 SL |
429 | |
430 | mir::Rvalue::UnaryOp(op, ref operand) => { | |
a1dfa0c6 | 431 | let operand = self.codegen_operand(&mut bx, operand); |
92a42be0 | 432 | let lloperand = operand.immediate(); |
dc9dc135 | 433 | let is_float = operand.layout.ty.is_floating_point(); |
92a42be0 | 434 | let llval = match op { |
2c00a5a8 | 435 | mir::UnOp::Not => bx.not(lloperand), |
dfeec247 XL |
436 | mir::UnOp::Neg => { |
437 | if is_float { | |
438 | bx.fneg(lloperand) | |
439 | } else { | |
440 | bx.neg(lloperand) | |
441 | } | |
92a42be0 SL |
442 | } |
443 | }; | |
dfeec247 | 444 | (bx, OperandRef { val: OperandValue::Immediate(llval), layout: operand.layout }) |
92a42be0 SL |
445 | } |
446 | ||
ff7c6d11 | 447 | mir::Rvalue::Discriminant(ref place) => { |
f9f354fc | 448 | let discr_ty = rvalue.ty(self.mir, bx.tcx()); |
fc512014 | 449 | let discr_ty = self.monomorphize(discr_ty); |
dfeec247 | 450 | let discr = self |
74b04a01 | 451 | .codegen_place(&mut bx, place.as_ref()) |
a1dfa0c6 | 452 | .codegen_get_discr(&mut bx, discr_ty); |
dfeec247 XL |
453 | ( |
454 | bx, | |
455 | OperandRef { | |
456 | val: OperandValue::Immediate(discr), | |
457 | layout: self.cx.layout_of(discr_ty), | |
458 | }, | |
459 | ) | |
8bb4bdeb XL |
460 | } |
461 | ||
c295e0f8 XL |
462 | mir::Rvalue::NullaryOp(null_op, ty) => { |
463 | let ty = self.monomorphize(ty); | |
464 | assert!(bx.cx().type_is_sized(ty)); | |
465 | let layout = bx.cx().layout_of(ty); | |
466 | let val = match null_op { | |
467 | mir::NullOp::SizeOf => layout.size.bytes(), | |
468 | mir::NullOp::AlignOf => layout.align.abi.bytes(), | |
c295e0f8 XL |
469 | }; |
470 | let val = bx.cx().const_usize(val); | |
471 | let tcx = self.cx.tcx(); | |
472 | ( | |
473 | bx, | |
474 | OperandRef { | |
475 | val: OperandValue::Immediate(val), | |
476 | layout: self.cx.layout_of(tcx.types.usize), | |
477 | }, | |
478 | ) | |
479 | } | |
480 | ||
f9f354fc XL |
481 | mir::Rvalue::ThreadLocalRef(def_id) => { |
482 | assert!(bx.cx().tcx().is_static(def_id)); | |
483 | let static_ = bx.get_static(def_id); | |
484 | let layout = bx.layout_of(bx.cx().tcx().static_ptr_ty(def_id)); | |
485 | let operand = OperandRef::from_immediate_or_packed_pair(&mut bx, static_, layout); | |
486 | (bx, operand) | |
487 | } | |
a7813a04 | 488 | mir::Rvalue::Use(ref operand) => { |
a1dfa0c6 | 489 | let operand = self.codegen_operand(&mut bx, operand); |
2c00a5a8 | 490 | (bx, operand) |
a7813a04 | 491 | } |
dfeec247 | 492 | mir::Rvalue::Repeat(..) | mir::Rvalue::Aggregate(..) => { |
cc61c64b XL |
493 | // According to `rvalue_creates_operand`, only ZST |
494 | // aggregate rvalues are allowed to be operands. | |
f9f354fc | 495 | let ty = rvalue.ty(self.mir, self.cx.tcx()); |
dfeec247 | 496 | let operand = |
fc512014 | 497 | OperandRef::new_zst(&mut bx, self.cx.layout_of(self.monomorphize(ty))); |
532ac7d7 | 498 | (bx, operand) |
92a42be0 | 499 | } |
c295e0f8 XL |
500 | mir::Rvalue::ShallowInitBox(ref operand, content_ty) => { |
501 | let operand = self.codegen_operand(&mut bx, operand); | |
502 | let lloperand = operand.immediate(); | |
503 | ||
504 | let content_ty = self.monomorphize(content_ty); | |
505 | let box_layout = bx.cx().layout_of(bx.tcx().mk_box(content_ty)); | |
506 | let llty_ptr = bx.cx().backend_type(box_layout); | |
507 | ||
508 | let val = bx.pointercast(lloperand, llty_ptr); | |
509 | let operand = OperandRef { val: OperandValue::Immediate(val), layout: box_layout }; | |
510 | (bx, operand) | |
511 | } | |
92a42be0 SL |
512 | } |
513 | } | |
514 | ||
ba9703b0 | 515 | fn evaluate_array_len(&mut self, bx: &mut Bx, place: mir::Place<'tcx>) -> Bx::Value { |
3b2f2976 | 516 | // ZST are passed as operands and require special handling |
94b46f34 | 517 | // because codegen_place() panics if Local is operand. |
e74abb32 | 518 | if let Some(index) = place.as_local() { |
3b2f2976 | 519 | if let LocalRef::Operand(Some(op)) = self.locals[index] { |
1b1a35ee | 520 | if let ty::Array(_, n) = op.layout.ty.kind() { |
416331ca | 521 | let n = n.eval_usize(bx.cx().tcx(), ty::ParamEnv::reveal_all()); |
a1dfa0c6 | 522 | return bx.cx().const_usize(n); |
3b2f2976 XL |
523 | } |
524 | } | |
525 | } | |
526 | // use common size calculation for non zero-sized types | |
74b04a01 | 527 | let cg_value = self.codegen_place(bx, place.as_ref()); |
416331ca | 528 | cg_value.len(bx.cx()) |
3b2f2976 XL |
529 | } |
530 | ||
dfeec247 XL |
531 | /// Codegen an `Rvalue::AddressOf` or `Rvalue::Ref` |
532 | fn codegen_place_to_pointer( | |
533 | &mut self, | |
534 | mut bx: Bx, | |
ba9703b0 | 535 | place: mir::Place<'tcx>, |
dfeec247 XL |
536 | mk_ptr_ty: impl FnOnce(TyCtxt<'tcx>, Ty<'tcx>) -> Ty<'tcx>, |
537 | ) -> (Bx, OperandRef<'tcx, Bx::Value>) { | |
74b04a01 | 538 | let cg_place = self.codegen_place(&mut bx, place.as_ref()); |
dfeec247 XL |
539 | |
540 | let ty = cg_place.layout.ty; | |
541 | ||
542 | // Note: places are indirect, so storing the `llval` into the | |
543 | // destination effectively creates a reference. | |
544 | let val = if !bx.cx().type_has_metadata(ty) { | |
545 | OperandValue::Immediate(cg_place.llval) | |
546 | } else { | |
547 | OperandValue::Pair(cg_place.llval, cg_place.llextra.unwrap()) | |
548 | }; | |
549 | (bx, OperandRef { val, layout: self.cx.layout_of(mk_ptr_ty(self.cx.tcx(), ty)) }) | |
550 | } | |
551 | ||
b7449926 XL |
552 | pub fn codegen_scalar_binop( |
553 | &mut self, | |
a1dfa0c6 | 554 | bx: &mut Bx, |
b7449926 | 555 | op: mir::BinOp, |
a1dfa0c6 XL |
556 | lhs: Bx::Value, |
557 | rhs: Bx::Value, | |
b7449926 | 558 | input_ty: Ty<'tcx>, |
a1dfa0c6 | 559 | ) -> Bx::Value { |
dc9dc135 | 560 | let is_float = input_ty.is_floating_point(); |
92a42be0 SL |
561 | let is_signed = input_ty.is_signed(); |
562 | match op { | |
dfeec247 XL |
563 | mir::BinOp::Add => { |
564 | if is_float { | |
565 | bx.fadd(lhs, rhs) | |
566 | } else { | |
567 | bx.add(lhs, rhs) | |
568 | } | |
569 | } | |
570 | mir::BinOp::Sub => { | |
571 | if is_float { | |
572 | bx.fsub(lhs, rhs) | |
573 | } else { | |
574 | bx.sub(lhs, rhs) | |
575 | } | |
576 | } | |
577 | mir::BinOp::Mul => { | |
578 | if is_float { | |
579 | bx.fmul(lhs, rhs) | |
580 | } else { | |
581 | bx.mul(lhs, rhs) | |
582 | } | |
583 | } | |
584 | mir::BinOp::Div => { | |
585 | if is_float { | |
586 | bx.fdiv(lhs, rhs) | |
587 | } else if is_signed { | |
588 | bx.sdiv(lhs, rhs) | |
589 | } else { | |
590 | bx.udiv(lhs, rhs) | |
591 | } | |
592 | } | |
593 | mir::BinOp::Rem => { | |
594 | if is_float { | |
595 | bx.frem(lhs, rhs) | |
596 | } else if is_signed { | |
597 | bx.srem(lhs, rhs) | |
598 | } else { | |
599 | bx.urem(lhs, rhs) | |
600 | } | |
601 | } | |
2c00a5a8 XL |
602 | mir::BinOp::BitOr => bx.or(lhs, rhs), |
603 | mir::BinOp::BitAnd => bx.and(lhs, rhs), | |
604 | mir::BinOp::BitXor => bx.xor(lhs, rhs), | |
94222f64 XL |
605 | mir::BinOp::Offset => { |
606 | let pointee_type = input_ty | |
607 | .builtin_deref(true) | |
608 | .unwrap_or_else(|| bug!("deref of non-pointer {:?}", input_ty)) | |
609 | .ty; | |
610 | let llty = bx.cx().backend_type(bx.cx().layout_of(pointee_type)); | |
611 | bx.inbounds_gep(llty, lhs, &[rhs]) | |
612 | } | |
2c00a5a8 XL |
613 | mir::BinOp::Shl => common::build_unchecked_lshift(bx, lhs, rhs), |
614 | mir::BinOp::Shr => common::build_unchecked_rshift(bx, input_ty, lhs, rhs), | |
dfeec247 XL |
615 | mir::BinOp::Ne |
616 | | mir::BinOp::Lt | |
617 | | mir::BinOp::Gt | |
618 | | mir::BinOp::Eq | |
619 | | mir::BinOp::Le | |
620 | | mir::BinOp::Ge => { | |
621 | if is_float { | |
622 | bx.fcmp(base::bin_op_to_fcmp_predicate(op.to_hir_binop()), lhs, rhs) | |
623 | } else { | |
624 | bx.icmp(base::bin_op_to_icmp_predicate(op.to_hir_binop(), is_signed), lhs, rhs) | |
625 | } | |
c30ab7b3 SL |
626 | } |
627 | } | |
628 | } | |
629 | ||
b7449926 XL |
630 | pub fn codegen_fat_ptr_binop( |
631 | &mut self, | |
a1dfa0c6 | 632 | bx: &mut Bx, |
b7449926 | 633 | op: mir::BinOp, |
a1dfa0c6 XL |
634 | lhs_addr: Bx::Value, |
635 | lhs_extra: Bx::Value, | |
636 | rhs_addr: Bx::Value, | |
637 | rhs_extra: Bx::Value, | |
b7449926 | 638 | _input_ty: Ty<'tcx>, |
a1dfa0c6 | 639 | ) -> Bx::Value { |
c30ab7b3 SL |
640 | match op { |
641 | mir::BinOp::Eq => { | |
a1dfa0c6 XL |
642 | let lhs = bx.icmp(IntPredicate::IntEQ, lhs_addr, rhs_addr); |
643 | let rhs = bx.icmp(IntPredicate::IntEQ, lhs_extra, rhs_extra); | |
644 | bx.and(lhs, rhs) | |
c30ab7b3 SL |
645 | } |
646 | mir::BinOp::Ne => { | |
a1dfa0c6 XL |
647 | let lhs = bx.icmp(IntPredicate::IntNE, lhs_addr, rhs_addr); |
648 | let rhs = bx.icmp(IntPredicate::IntNE, lhs_extra, rhs_extra); | |
649 | bx.or(lhs, rhs) | |
c30ab7b3 | 650 | } |
dfeec247 | 651 | mir::BinOp::Le | mir::BinOp::Lt | mir::BinOp::Ge | mir::BinOp::Gt => { |
c30ab7b3 SL |
652 | // a OP b ~ a.0 STRICT(OP) b.0 | (a.0 == b.0 && a.1 OP a.1) |
653 | let (op, strict_op) = match op { | |
a1dfa0c6 XL |
654 | mir::BinOp::Lt => (IntPredicate::IntULT, IntPredicate::IntULT), |
655 | mir::BinOp::Le => (IntPredicate::IntULE, IntPredicate::IntULT), | |
656 | mir::BinOp::Gt => (IntPredicate::IntUGT, IntPredicate::IntUGT), | |
657 | mir::BinOp::Ge => (IntPredicate::IntUGE, IntPredicate::IntUGT), | |
c30ab7b3 SL |
658 | _ => bug!(), |
659 | }; | |
a1dfa0c6 XL |
660 | let lhs = bx.icmp(strict_op, lhs_addr, rhs_addr); |
661 | let and_lhs = bx.icmp(IntPredicate::IntEQ, lhs_addr, rhs_addr); | |
662 | let and_rhs = bx.icmp(op, lhs_extra, rhs_extra); | |
663 | let rhs = bx.and(and_lhs, and_rhs); | |
664 | bx.or(lhs, rhs) | |
c30ab7b3 SL |
665 | } |
666 | _ => { | |
667 | bug!("unexpected fat ptr binop"); | |
92a42be0 SL |
668 | } |
669 | } | |
670 | } | |
3157f602 | 671 | |
a1dfa0c6 XL |
672 | pub fn codegen_scalar_checked_binop( |
673 | &mut self, | |
674 | bx: &mut Bx, | |
675 | op: mir::BinOp, | |
676 | lhs: Bx::Value, | |
677 | rhs: Bx::Value, | |
dfeec247 | 678 | input_ty: Ty<'tcx>, |
a1dfa0c6 | 679 | ) -> OperandValue<Bx::Value> { |
3157f602 XL |
680 | // This case can currently arise only from functions marked |
681 | // with #[rustc_inherit_overflow_checks] and inlined from | |
682 | // another crate (mostly core::num generic/#[inline] fns), | |
683 | // while the current crate doesn't use overflow checks. | |
a1dfa0c6 | 684 | if !bx.cx().check_overflow() { |
94b46f34 | 685 | let val = self.codegen_scalar_binop(bx, op, lhs, rhs, input_ty); |
a1dfa0c6 | 686 | return OperandValue::Pair(val, bx.cx().const_bool(false)); |
3157f602 XL |
687 | } |
688 | ||
3157f602 XL |
689 | let (val, of) = match op { |
690 | // These are checked using intrinsics | |
691 | mir::BinOp::Add | mir::BinOp::Sub | mir::BinOp::Mul => { | |
692 | let oop = match op { | |
693 | mir::BinOp::Add => OverflowOp::Add, | |
694 | mir::BinOp::Sub => OverflowOp::Sub, | |
695 | mir::BinOp::Mul => OverflowOp::Mul, | |
dfeec247 | 696 | _ => unreachable!(), |
3157f602 | 697 | }; |
a1dfa0c6 | 698 | bx.checked_binop(oop, input_ty, lhs, rhs) |
3157f602 XL |
699 | } |
700 | mir::BinOp::Shl | mir::BinOp::Shr => { | |
a1dfa0c6 XL |
701 | let lhs_llty = bx.cx().val_ty(lhs); |
702 | let rhs_llty = bx.cx().val_ty(rhs); | |
703 | let invert_mask = common::shift_mask_val(bx, lhs_llty, rhs_llty, true); | |
2c00a5a8 | 704 | let outer_bits = bx.and(rhs, invert_mask); |
3157f602 | 705 | |
a1dfa0c6 | 706 | let of = bx.icmp(IntPredicate::IntNE, outer_bits, bx.cx().const_null(rhs_llty)); |
94b46f34 | 707 | let val = self.codegen_scalar_binop(bx, op, lhs, rhs, input_ty); |
3157f602 XL |
708 | |
709 | (val, of) | |
710 | } | |
dfeec247 | 711 | _ => bug!("Operator `{:?}` is not a checkable operator", op), |
3157f602 XL |
712 | }; |
713 | ||
714 | OperandValue::Pair(val, of) | |
715 | } | |
a1dfa0c6 | 716 | } |
92a42be0 | 717 | |
dc9dc135 | 718 | impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { |
416331ca | 719 | pub fn rvalue_creates_operand(&self, rvalue: &mir::Rvalue<'tcx>, span: Span) -> bool { |
cc61c64b XL |
720 | match *rvalue { |
721 | mir::Rvalue::Ref(..) | | |
064997fb | 722 | mir::Rvalue::CopyForDeref(..) | |
dfeec247 | 723 | mir::Rvalue::AddressOf(..) | |
cc61c64b XL |
724 | mir::Rvalue::Len(..) | |
725 | mir::Rvalue::Cast(..) | // (*) | |
c295e0f8 | 726 | mir::Rvalue::ShallowInitBox(..) | // (*) |
cc61c64b XL |
727 | mir::Rvalue::BinaryOp(..) | |
728 | mir::Rvalue::CheckedBinaryOp(..) | | |
729 | mir::Rvalue::UnaryOp(..) | | |
730 | mir::Rvalue::Discriminant(..) | | |
7cac9316 | 731 | mir::Rvalue::NullaryOp(..) | |
f9f354fc | 732 | mir::Rvalue::ThreadLocalRef(_) | |
cc61c64b XL |
733 | mir::Rvalue::Use(..) => // (*) |
734 | true, | |
735 | mir::Rvalue::Repeat(..) | | |
736 | mir::Rvalue::Aggregate(..) => { | |
f9f354fc | 737 | let ty = rvalue.ty(self.mir, self.cx.tcx()); |
fc512014 | 738 | let ty = self.monomorphize(ty); |
416331ca | 739 | self.cx.spanned_layout_of(ty, span).is_zst() |
cc61c64b XL |
740 | } |
741 | } | |
92a42be0 | 742 | |
cc61c64b XL |
743 | // (*) this is only true if the type is suitable |
744 | } | |
92a42be0 | 745 | } |