]>
Commit | Line | Data |
---|---|---|
a2a8927a | 1 | //! Codegen of `asm!` invocations. |
29967ef6 XL |
2 | |
3 | use crate::prelude::*; | |
4 | ||
5 | use std::fmt::Write; | |
6 | ||
7 | use rustc_ast::ast::{InlineAsmOptions, InlineAsmTemplatePiece}; | |
8 | use rustc_middle::mir::InlineAsmOperand; | |
5099ac24 | 9 | use rustc_span::sym; |
29967ef6 XL |
10 | use rustc_target::asm::*; |
11 | ||
9ffffee4 FG |
12 | enum CInlineAsmOperand<'tcx> { |
13 | In { | |
14 | reg: InlineAsmRegOrRegClass, | |
15 | value: CValue<'tcx>, | |
16 | }, | |
17 | Out { | |
18 | reg: InlineAsmRegOrRegClass, | |
19 | late: bool, | |
20 | place: Option<CPlace<'tcx>>, | |
21 | }, | |
22 | InOut { | |
23 | reg: InlineAsmRegOrRegClass, | |
24 | _late: bool, | |
25 | in_value: CValue<'tcx>, | |
26 | out_place: Option<CPlace<'tcx>>, | |
27 | }, | |
28 | Const { | |
29 | value: String, | |
30 | }, | |
31 | Symbol { | |
32 | symbol: String, | |
33 | }, | |
34 | } | |
35 | ||
29967ef6 | 36 | pub(crate) fn codegen_inline_asm<'tcx>( |
6a06907d | 37 | fx: &mut FunctionCx<'_, '_, 'tcx>, |
9ffffee4 | 38 | span: Span, |
29967ef6 XL |
39 | template: &[InlineAsmTemplatePiece], |
40 | operands: &[InlineAsmOperand<'tcx>], | |
41 | options: InlineAsmOptions, | |
f2b60f7d | 42 | destination: Option<mir::BasicBlock>, |
29967ef6 XL |
43 | ) { |
44 | // FIXME add .eh_frame unwind info directives | |
45 | ||
064997fb | 46 | if !template.is_empty() { |
f2b60f7d | 47 | // Used by panic_abort |
064997fb | 48 | if template[0] == InlineAsmTemplatePiece::String("int $$0x29".to_string()) { |
f2b60f7d | 49 | fx.bcx.ins().trap(TrapCode::User(1)); |
064997fb | 50 | return; |
f2b60f7d FG |
51 | } |
52 | ||
53 | // Used by stdarch | |
2b03887a | 54 | if template[0] == InlineAsmTemplatePiece::String("mov ".to_string()) |
064997fb FG |
55 | && matches!( |
56 | template[1], | |
57 | InlineAsmTemplatePiece::Placeholder { | |
58 | operand_idx: 0, | |
59 | modifier: Some('r'), | |
60 | span: _ | |
61 | } | |
62 | ) | |
2b03887a FG |
63 | && template[2] == InlineAsmTemplatePiece::String(", rbx".to_string()) |
64 | && template[3] == InlineAsmTemplatePiece::String("\n".to_string()) | |
65 | && template[4] == InlineAsmTemplatePiece::String("cpuid".to_string()) | |
66 | && template[5] == InlineAsmTemplatePiece::String("\n".to_string()) | |
67 | && template[6] == InlineAsmTemplatePiece::String("xchg ".to_string()) | |
064997fb | 68 | && matches!( |
2b03887a | 69 | template[7], |
064997fb FG |
70 | InlineAsmTemplatePiece::Placeholder { |
71 | operand_idx: 0, | |
72 | modifier: Some('r'), | |
73 | span: _ | |
74 | } | |
75 | ) | |
2b03887a | 76 | && template[8] == InlineAsmTemplatePiece::String(", rbx".to_string()) |
064997fb FG |
77 | { |
78 | assert_eq!(operands.len(), 4); | |
79 | let (leaf, eax_place) = match operands[1] { | |
f2b60f7d FG |
80 | InlineAsmOperand::InOut { |
81 | reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::ax)), | |
2b03887a | 82 | late: _, |
f2b60f7d FG |
83 | ref in_value, |
84 | out_place: Some(out_place), | |
85 | } => ( | |
86 | crate::base::codegen_operand(fx, in_value).load_scalar(fx), | |
87 | crate::base::codegen_place(fx, out_place), | |
88 | ), | |
064997fb FG |
89 | _ => unreachable!(), |
90 | }; | |
91 | let ebx_place = match operands[0] { | |
f2b60f7d FG |
92 | InlineAsmOperand::Out { |
93 | reg: | |
064997fb | 94 | InlineAsmRegOrRegClass::RegClass(InlineAsmRegClass::X86( |
f2b60f7d FG |
95 | X86InlineAsmRegClass::reg, |
96 | )), | |
2b03887a | 97 | late: _, |
f2b60f7d FG |
98 | place: Some(place), |
99 | } => crate::base::codegen_place(fx, place), | |
064997fb FG |
100 | _ => unreachable!(), |
101 | }; | |
102 | let (sub_leaf, ecx_place) = match operands[2] { | |
f2b60f7d FG |
103 | InlineAsmOperand::InOut { |
104 | reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::cx)), | |
2b03887a | 105 | late: _, |
f2b60f7d FG |
106 | ref in_value, |
107 | out_place: Some(out_place), | |
108 | } => ( | |
109 | crate::base::codegen_operand(fx, in_value).load_scalar(fx), | |
110 | crate::base::codegen_place(fx, out_place), | |
111 | ), | |
064997fb FG |
112 | _ => unreachable!(), |
113 | }; | |
114 | let edx_place = match operands[3] { | |
f2b60f7d FG |
115 | InlineAsmOperand::Out { |
116 | reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::dx)), | |
2b03887a | 117 | late: _, |
f2b60f7d FG |
118 | place: Some(place), |
119 | } => crate::base::codegen_place(fx, place), | |
064997fb FG |
120 | _ => unreachable!(), |
121 | }; | |
17df50a5 | 122 | |
064997fb | 123 | let (eax, ebx, ecx, edx) = crate::intrinsics::codegen_cpuid_call(fx, leaf, sub_leaf); |
17df50a5 | 124 | |
064997fb FG |
125 | eax_place.write_cvalue(fx, CValue::by_val(eax, fx.layout_of(fx.tcx.types.u32))); |
126 | ebx_place.write_cvalue(fx, CValue::by_val(ebx, fx.layout_of(fx.tcx.types.u32))); | |
127 | ecx_place.write_cvalue(fx, CValue::by_val(ecx, fx.layout_of(fx.tcx.types.u32))); | |
128 | edx_place.write_cvalue(fx, CValue::by_val(edx, fx.layout_of(fx.tcx.types.u32))); | |
f2b60f7d FG |
129 | let destination_block = fx.get_block(destination.unwrap()); |
130 | fx.bcx.ins().jump(destination_block, &[]); | |
064997fb | 131 | return; |
f2b60f7d FG |
132 | } |
133 | ||
134 | // Used by compiler-builtins | |
135 | if fx.tcx.symbol_name(fx.instance).name.starts_with("___chkstk") { | |
064997fb FG |
136 | // ___chkstk, ___chkstk_ms and __alloca are only used on Windows |
137 | crate::trap::trap_unimplemented(fx, "Stack probes are not supported"); | |
f2b60f7d | 138 | return; |
064997fb FG |
139 | } else if fx.tcx.symbol_name(fx.instance).name == "__alloca" { |
140 | crate::trap::trap_unimplemented(fx, "Alloca is not supported"); | |
f2b60f7d FG |
141 | return; |
142 | } | |
143 | ||
144 | // Used by measureme | |
145 | if template[0] == InlineAsmTemplatePiece::String("xor %eax, %eax".to_string()) | |
146 | && template[1] == InlineAsmTemplatePiece::String("\n".to_string()) | |
147 | && template[2] == InlineAsmTemplatePiece::String("mov %rbx, ".to_string()) | |
148 | && matches!( | |
149 | template[3], | |
150 | InlineAsmTemplatePiece::Placeholder { | |
151 | operand_idx: 0, | |
152 | modifier: Some('r'), | |
153 | span: _ | |
154 | } | |
155 | ) | |
156 | && template[4] == InlineAsmTemplatePiece::String("\n".to_string()) | |
157 | && template[5] == InlineAsmTemplatePiece::String("cpuid".to_string()) | |
158 | && template[6] == InlineAsmTemplatePiece::String("\n".to_string()) | |
159 | && template[7] == InlineAsmTemplatePiece::String("mov ".to_string()) | |
160 | && matches!( | |
161 | template[8], | |
162 | InlineAsmTemplatePiece::Placeholder { | |
163 | operand_idx: 0, | |
164 | modifier: Some('r'), | |
165 | span: _ | |
166 | } | |
167 | ) | |
168 | && template[9] == InlineAsmTemplatePiece::String(", %rbx".to_string()) | |
169 | { | |
170 | let destination_block = fx.get_block(destination.unwrap()); | |
171 | fx.bcx.ins().jump(destination_block, &[]); | |
172 | return; | |
173 | } else if template[0] == InlineAsmTemplatePiece::String("rdpmc".to_string()) { | |
174 | // Return zero dummy values for all performance counters | |
175 | match operands[0] { | |
176 | InlineAsmOperand::In { | |
177 | reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::cx)), | |
178 | value: _, | |
179 | } => {} | |
180 | _ => unreachable!(), | |
181 | }; | |
182 | let lo = match operands[1] { | |
183 | InlineAsmOperand::Out { | |
184 | reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::ax)), | |
185 | late: true, | |
186 | place: Some(place), | |
187 | } => crate::base::codegen_place(fx, place), | |
188 | _ => unreachable!(), | |
189 | }; | |
190 | let hi = match operands[2] { | |
191 | InlineAsmOperand::Out { | |
192 | reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::dx)), | |
193 | late: true, | |
194 | place: Some(place), | |
195 | } => crate::base::codegen_place(fx, place), | |
196 | _ => unreachable!(), | |
197 | }; | |
198 | ||
199 | let u32_layout = fx.layout_of(fx.tcx.types.u32); | |
200 | let zero = fx.bcx.ins().iconst(types::I32, 0); | |
201 | lo.write_cvalue(fx, CValue::by_val(zero, u32_layout)); | |
202 | hi.write_cvalue(fx, CValue::by_val(zero, u32_layout)); | |
203 | ||
204 | let destination_block = fx.get_block(destination.unwrap()); | |
205 | fx.bcx.ins().jump(destination_block, &[]); | |
206 | return; | |
207 | } else if template[0] == InlineAsmTemplatePiece::String("lock xadd ".to_string()) | |
208 | && matches!( | |
209 | template[1], | |
210 | InlineAsmTemplatePiece::Placeholder { operand_idx: 1, modifier: None, span: _ } | |
211 | ) | |
212 | && template[2] == InlineAsmTemplatePiece::String(", (".to_string()) | |
213 | && matches!( | |
214 | template[3], | |
215 | InlineAsmTemplatePiece::Placeholder { operand_idx: 0, modifier: None, span: _ } | |
216 | ) | |
217 | && template[4] == InlineAsmTemplatePiece::String(")".to_string()) | |
218 | { | |
219 | let destination_block = fx.get_block(destination.unwrap()); | |
220 | fx.bcx.ins().jump(destination_block, &[]); | |
221 | return; | |
064997fb | 222 | } |
29967ef6 XL |
223 | } |
224 | ||
9ffffee4 FG |
225 | let operands = operands |
226 | .into_iter() | |
227 | .map(|operand| match *operand { | |
228 | InlineAsmOperand::In { reg, ref value } => { | |
229 | CInlineAsmOperand::In { reg, value: crate::base::codegen_operand(fx, value) } | |
230 | } | |
231 | InlineAsmOperand::Out { reg, late, ref place } => CInlineAsmOperand::Out { | |
232 | reg, | |
233 | late, | |
234 | place: place.map(|place| crate::base::codegen_place(fx, place)), | |
235 | }, | |
236 | InlineAsmOperand::InOut { reg, late, ref in_value, ref out_place } => { | |
237 | CInlineAsmOperand::InOut { | |
238 | reg, | |
239 | _late: late, | |
240 | in_value: crate::base::codegen_operand(fx, in_value), | |
241 | out_place: out_place.map(|place| crate::base::codegen_place(fx, place)), | |
242 | } | |
243 | } | |
244 | InlineAsmOperand::Const { ref value } => { | |
781aab86 | 245 | let (const_value, ty) = crate::constant::eval_mir_constant(fx, value); |
9ffffee4 FG |
246 | let value = rustc_codegen_ssa::common::asm_const_to_str( |
247 | fx.tcx, | |
248 | span, | |
249 | const_value, | |
250 | fx.layout_of(ty), | |
251 | ); | |
252 | CInlineAsmOperand::Const { value } | |
253 | } | |
254 | InlineAsmOperand::SymFn { ref value } => { | |
781aab86 FG |
255 | let const_ = fx.monomorphize(value.const_); |
256 | if let ty::FnDef(def_id, args) = *const_.ty().kind() { | |
9ffffee4 FG |
257 | let instance = ty::Instance::resolve_for_fn_ptr( |
258 | fx.tcx, | |
259 | ty::ParamEnv::reveal_all(), | |
260 | def_id, | |
add651ee | 261 | args, |
9ffffee4 FG |
262 | ) |
263 | .unwrap(); | |
264 | let symbol = fx.tcx.symbol_name(instance); | |
265 | ||
266 | // Pass a wrapper rather than the function itself as the function itself may not | |
267 | // be exported from the main codegen unit and may thus be unreachable from the | |
268 | // object file created by an external assembler. | |
269 | let inline_asm_index = fx.cx.inline_asm_index.get(); | |
270 | fx.cx.inline_asm_index.set(inline_asm_index + 1); | |
271 | let wrapper_name = format!( | |
272 | "__inline_asm_{}_wrapper_n{}", | |
273 | fx.cx.cgu_name.as_str().replace('.', "__").replace('-', "_"), | |
274 | inline_asm_index | |
275 | ); | |
276 | let sig = | |
277 | get_function_sig(fx.tcx, fx.target_config.default_call_conv, instance); | |
278 | create_wrapper_function( | |
279 | fx.module, | |
280 | &mut fx.cx.unwind_context, | |
281 | sig, | |
282 | &wrapper_name, | |
283 | symbol.name, | |
284 | ); | |
285 | ||
286 | CInlineAsmOperand::Symbol { symbol: wrapper_name } | |
287 | } else { | |
288 | span_bug!(span, "invalid type for asm sym (fn)"); | |
289 | } | |
290 | } | |
291 | InlineAsmOperand::SymStatic { def_id } => { | |
292 | assert!(fx.tcx.is_static(def_id)); | |
293 | let instance = Instance::mono(fx.tcx, def_id).polymorphize(fx.tcx); | |
294 | CInlineAsmOperand::Symbol { symbol: fx.tcx.symbol_name(instance).name.to_owned() } | |
295 | } | |
296 | }) | |
297 | .collect::<Vec<_>>(); | |
298 | ||
29967ef6 XL |
299 | let mut inputs = Vec::new(); |
300 | let mut outputs = Vec::new(); | |
301 | ||
a2a8927a XL |
302 | let mut asm_gen = InlineAssemblyGenerator { |
303 | tcx: fx.tcx, | |
304 | arch: fx.tcx.sess.asm_arch.unwrap(), | |
5e7ed085 | 305 | enclosing_def_id: fx.instance.def_id(), |
a2a8927a | 306 | template, |
9ffffee4 | 307 | operands: &operands, |
a2a8927a XL |
308 | options, |
309 | registers: Vec::new(), | |
310 | stack_slots_clobber: Vec::new(), | |
311 | stack_slots_input: Vec::new(), | |
312 | stack_slots_output: Vec::new(), | |
313 | stack_slot_size: Size::from_bytes(0), | |
29967ef6 | 314 | }; |
a2a8927a XL |
315 | asm_gen.allocate_registers(); |
316 | asm_gen.allocate_stack_slots(); | |
317 | ||
318 | let inline_asm_index = fx.cx.inline_asm_index.get(); | |
319 | fx.cx.inline_asm_index.set(inline_asm_index + 1); | |
320 | let asm_name = format!( | |
321 | "__inline_asm_{}_n{}", | |
322 | fx.cx.cgu_name.as_str().replace('.', "__").replace('-', "_"), | |
323 | inline_asm_index | |
324 | ); | |
325 | ||
326 | let generated_asm = asm_gen.generate_asm_wrapper(&asm_name); | |
327 | fx.cx.global_asm.push_str(&generated_asm); | |
29967ef6 | 328 | |
a2a8927a | 329 | for (i, operand) in operands.iter().enumerate() { |
9ffffee4 FG |
330 | match operand { |
331 | CInlineAsmOperand::In { reg: _, value } => { | |
332 | inputs.push((asm_gen.stack_slots_input[i].unwrap(), value.load_scalar(fx))); | |
29967ef6 | 333 | } |
9ffffee4 | 334 | CInlineAsmOperand::Out { reg: _, late: _, place } => { |
29967ef6 | 335 | if let Some(place) = place { |
353b0b11 | 336 | outputs.push((asm_gen.stack_slots_output[i].unwrap(), *place)); |
29967ef6 XL |
337 | } |
338 | } | |
9ffffee4 FG |
339 | CInlineAsmOperand::InOut { reg: _, _late: _, in_value, out_place } => { |
340 | inputs.push((asm_gen.stack_slots_input[i].unwrap(), in_value.load_scalar(fx))); | |
29967ef6 | 341 | if let Some(out_place) = out_place { |
353b0b11 | 342 | outputs.push((asm_gen.stack_slots_output[i].unwrap(), *out_place)); |
29967ef6 XL |
343 | } |
344 | } | |
9ffffee4 | 345 | CInlineAsmOperand::Const { value: _ } | CInlineAsmOperand::Symbol { symbol: _ } => {} |
29967ef6 XL |
346 | } |
347 | } | |
348 | ||
a2a8927a | 349 | call_inline_asm(fx, &asm_name, asm_gen.stack_slot_size, inputs, outputs); |
f2b60f7d FG |
350 | |
351 | match destination { | |
352 | Some(destination) => { | |
353 | let destination_block = fx.get_block(destination); | |
354 | fx.bcx.ins().jump(destination_block, &[]); | |
355 | } | |
356 | None => { | |
357 | fx.bcx.ins().trap(TrapCode::UnreachableCodeReached); | |
358 | } | |
359 | } | |
29967ef6 XL |
360 | } |
361 | ||
a2a8927a XL |
362 | struct InlineAssemblyGenerator<'a, 'tcx> { |
363 | tcx: TyCtxt<'tcx>, | |
29967ef6 | 364 | arch: InlineAsmArch, |
5e7ed085 | 365 | enclosing_def_id: DefId, |
a2a8927a | 366 | template: &'a [InlineAsmTemplatePiece], |
9ffffee4 | 367 | operands: &'a [CInlineAsmOperand<'tcx>], |
29967ef6 | 368 | options: InlineAsmOptions, |
a2a8927a XL |
369 | registers: Vec<Option<InlineAsmReg>>, |
370 | stack_slots_clobber: Vec<Option<Size>>, | |
371 | stack_slots_input: Vec<Option<Size>>, | |
372 | stack_slots_output: Vec<Option<Size>>, | |
373 | stack_slot_size: Size, | |
374 | } | |
375 | ||
376 | impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> { | |
377 | fn allocate_registers(&mut self) { | |
378 | let sess = self.tcx.sess; | |
5e7ed085 FG |
379 | let map = allocatable_registers( |
380 | self.arch, | |
381 | sess.relocation_model(), | |
382 | self.tcx.asm_target_features(self.enclosing_def_id), | |
383 | &sess.target, | |
384 | ); | |
a2a8927a XL |
385 | let mut allocated = FxHashMap::<_, (bool, bool)>::default(); |
386 | let mut regs = vec![None; self.operands.len()]; | |
387 | ||
388 | // Add explicit registers to the allocated set. | |
389 | for (i, operand) in self.operands.iter().enumerate() { | |
390 | match *operand { | |
9ffffee4 | 391 | CInlineAsmOperand::In { reg: InlineAsmRegOrRegClass::Reg(reg), .. } => { |
a2a8927a XL |
392 | regs[i] = Some(reg); |
393 | allocated.entry(reg).or_default().0 = true; | |
394 | } | |
9ffffee4 FG |
395 | CInlineAsmOperand::Out { |
396 | reg: InlineAsmRegOrRegClass::Reg(reg), | |
397 | late: true, | |
398 | .. | |
a2a8927a XL |
399 | } => { |
400 | regs[i] = Some(reg); | |
401 | allocated.entry(reg).or_default().1 = true; | |
402 | } | |
9ffffee4 FG |
403 | CInlineAsmOperand::Out { reg: InlineAsmRegOrRegClass::Reg(reg), .. } |
404 | | CInlineAsmOperand::InOut { reg: InlineAsmRegOrRegClass::Reg(reg), .. } => { | |
a2a8927a XL |
405 | regs[i] = Some(reg); |
406 | allocated.insert(reg, (true, true)); | |
407 | } | |
408 | _ => (), | |
409 | } | |
29967ef6 | 410 | } |
29967ef6 | 411 | |
a2a8927a XL |
412 | // Allocate out/inout/inlateout registers first because they are more constrained. |
413 | for (i, operand) in self.operands.iter().enumerate() { | |
414 | match *operand { | |
9ffffee4 | 415 | CInlineAsmOperand::Out { |
a2a8927a XL |
416 | reg: InlineAsmRegOrRegClass::RegClass(class), |
417 | late: false, | |
418 | .. | |
419 | } | |
9ffffee4 | 420 | | CInlineAsmOperand::InOut { |
a2a8927a XL |
421 | reg: InlineAsmRegOrRegClass::RegClass(class), .. |
422 | } => { | |
423 | let mut alloc_reg = None; | |
424 | for ® in &map[&class] { | |
425 | let mut used = false; | |
426 | reg.overlapping_regs(|r| { | |
427 | if allocated.contains_key(&r) { | |
428 | used = true; | |
429 | } | |
430 | }); | |
431 | ||
432 | if !used { | |
433 | alloc_reg = Some(reg); | |
434 | break; | |
435 | } | |
436 | } | |
437 | ||
438 | let reg = alloc_reg.expect("cannot allocate registers"); | |
439 | regs[i] = Some(reg); | |
440 | allocated.insert(reg, (true, true)); | |
441 | } | |
442 | _ => (), | |
443 | } | |
444 | } | |
445 | ||
446 | // Allocate in/lateout. | |
447 | for (i, operand) in self.operands.iter().enumerate() { | |
448 | match *operand { | |
9ffffee4 | 449 | CInlineAsmOperand::In { reg: InlineAsmRegOrRegClass::RegClass(class), .. } => { |
a2a8927a XL |
450 | let mut alloc_reg = None; |
451 | for ® in &map[&class] { | |
452 | let mut used = false; | |
453 | reg.overlapping_regs(|r| { | |
454 | if allocated.get(&r).copied().unwrap_or_default().0 { | |
455 | used = true; | |
456 | } | |
457 | }); | |
458 | ||
459 | if !used { | |
460 | alloc_reg = Some(reg); | |
461 | break; | |
462 | } | |
463 | } | |
464 | ||
465 | let reg = alloc_reg.expect("cannot allocate registers"); | |
466 | regs[i] = Some(reg); | |
467 | allocated.entry(reg).or_default().0 = true; | |
468 | } | |
9ffffee4 | 469 | CInlineAsmOperand::Out { |
a2a8927a XL |
470 | reg: InlineAsmRegOrRegClass::RegClass(class), |
471 | late: true, | |
472 | .. | |
473 | } => { | |
474 | let mut alloc_reg = None; | |
475 | for ® in &map[&class] { | |
476 | let mut used = false; | |
477 | reg.overlapping_regs(|r| { | |
478 | if allocated.get(&r).copied().unwrap_or_default().1 { | |
479 | used = true; | |
480 | } | |
481 | }); | |
482 | ||
483 | if !used { | |
484 | alloc_reg = Some(reg); | |
485 | break; | |
486 | } | |
487 | } | |
488 | ||
489 | let reg = alloc_reg.expect("cannot allocate registers"); | |
490 | regs[i] = Some(reg); | |
491 | allocated.entry(reg).or_default().1 = true; | |
492 | } | |
493 | _ => (), | |
494 | } | |
495 | } | |
29967ef6 | 496 | |
a2a8927a | 497 | self.registers = regs; |
29967ef6 XL |
498 | } |
499 | ||
a2a8927a XL |
500 | fn allocate_stack_slots(&mut self) { |
501 | let mut slot_size = Size::from_bytes(0); | |
502 | let mut slots_clobber = vec![None; self.operands.len()]; | |
503 | let mut slots_input = vec![None; self.operands.len()]; | |
504 | let mut slots_output = vec![None; self.operands.len()]; | |
505 | ||
506 | let new_slot_fn = |slot_size: &mut Size, reg_class: InlineAsmRegClass| { | |
507 | let reg_size = | |
508 | reg_class.supported_types(self.arch).iter().map(|(ty, _)| ty.size()).max().unwrap(); | |
509 | let align = rustc_target::abi::Align::from_bytes(reg_size.bytes()).unwrap(); | |
510 | let offset = slot_size.align_to(align); | |
511 | *slot_size = offset + reg_size; | |
512 | offset | |
513 | }; | |
514 | let mut new_slot = |x| new_slot_fn(&mut slot_size, x); | |
515 | ||
516 | // Allocate stack slots for saving clobbered registers | |
5e7ed085 FG |
517 | let abi_clobber = InlineAsmClobberAbi::parse(self.arch, &self.tcx.sess.target, sym::C) |
518 | .unwrap() | |
519 | .clobbered_regs(); | |
a2a8927a XL |
520 | for (i, reg) in self.registers.iter().enumerate().filter_map(|(i, r)| r.map(|r| (i, r))) { |
521 | let mut need_save = true; | |
522 | // If the register overlaps with a register clobbered by function call, then | |
523 | // we don't need to save it. | |
524 | for r in abi_clobber { | |
525 | r.overlapping_regs(|r| { | |
526 | if r == reg { | |
527 | need_save = false; | |
528 | } | |
529 | }); | |
530 | ||
531 | if !need_save { | |
532 | break; | |
533 | } | |
534 | } | |
535 | ||
536 | if need_save { | |
537 | slots_clobber[i] = Some(new_slot(reg.reg_class())); | |
29967ef6 | 538 | } |
29967ef6 | 539 | } |
a2a8927a XL |
540 | |
541 | // Allocate stack slots for inout | |
542 | for (i, operand) in self.operands.iter().enumerate() { | |
543 | match *operand { | |
9ffffee4 | 544 | CInlineAsmOperand::InOut { reg, out_place: Some(_), .. } => { |
a2a8927a XL |
545 | let slot = new_slot(reg.reg_class()); |
546 | slots_input[i] = Some(slot); | |
547 | slots_output[i] = Some(slot); | |
548 | } | |
549 | _ => (), | |
550 | } | |
551 | } | |
552 | ||
553 | let slot_size_before_input = slot_size; | |
554 | let mut new_slot = |x| new_slot_fn(&mut slot_size, x); | |
555 | ||
556 | // Allocate stack slots for input | |
557 | for (i, operand) in self.operands.iter().enumerate() { | |
558 | match *operand { | |
9ffffee4 FG |
559 | CInlineAsmOperand::In { reg, .. } |
560 | | CInlineAsmOperand::InOut { reg, out_place: None, .. } => { | |
a2a8927a XL |
561 | slots_input[i] = Some(new_slot(reg.reg_class())); |
562 | } | |
563 | _ => (), | |
564 | } | |
565 | } | |
566 | ||
567 | // Reset slot size to before input so that input and output operands can overlap | |
568 | // and save some memory. | |
569 | let slot_size_after_input = slot_size; | |
570 | slot_size = slot_size_before_input; | |
571 | let mut new_slot = |x| new_slot_fn(&mut slot_size, x); | |
572 | ||
573 | // Allocate stack slots for output | |
574 | for (i, operand) in self.operands.iter().enumerate() { | |
575 | match *operand { | |
9ffffee4 | 576 | CInlineAsmOperand::Out { reg, place: Some(_), .. } => { |
a2a8927a XL |
577 | slots_output[i] = Some(new_slot(reg.reg_class())); |
578 | } | |
579 | _ => (), | |
580 | } | |
581 | } | |
582 | ||
583 | slot_size = slot_size.max(slot_size_after_input); | |
584 | ||
585 | self.stack_slots_clobber = slots_clobber; | |
586 | self.stack_slots_input = slots_input; | |
587 | self.stack_slots_output = slots_output; | |
588 | self.stack_slot_size = slot_size; | |
29967ef6 | 589 | } |
29967ef6 | 590 | |
a2a8927a XL |
591 | fn generate_asm_wrapper(&self, asm_name: &str) -> String { |
592 | let mut generated_asm = String::new(); | |
593 | writeln!(generated_asm, ".globl {}", asm_name).unwrap(); | |
594 | writeln!(generated_asm, ".type {},@function", asm_name).unwrap(); | |
595 | writeln!(generated_asm, ".section .text.{},\"ax\",@progbits", asm_name).unwrap(); | |
596 | writeln!(generated_asm, "{}:", asm_name).unwrap(); | |
597 | ||
598 | let is_x86 = matches!(self.arch, InlineAsmArch::X86 | InlineAsmArch::X86_64); | |
599 | ||
600 | if is_x86 { | |
601 | generated_asm.push_str(".intel_syntax noprefix\n"); | |
602 | } | |
603 | Self::prologue(&mut generated_asm, self.arch); | |
604 | ||
605 | // Save clobbered registers | |
606 | if !self.options.contains(InlineAsmOptions::NORETURN) { | |
607 | for (reg, slot) in self | |
608 | .registers | |
609 | .iter() | |
610 | .zip(self.stack_slots_clobber.iter().copied()) | |
611 | .filter_map(|(r, s)| r.zip(s)) | |
612 | { | |
613 | Self::save_register(&mut generated_asm, self.arch, reg, slot); | |
614 | } | |
615 | } | |
616 | ||
617 | // Write input registers | |
618 | for (reg, slot) in self | |
619 | .registers | |
620 | .iter() | |
621 | .zip(self.stack_slots_input.iter().copied()) | |
622 | .filter_map(|(r, s)| r.zip(s)) | |
623 | { | |
624 | Self::restore_register(&mut generated_asm, self.arch, reg, slot); | |
625 | } | |
626 | ||
627 | if is_x86 && self.options.contains(InlineAsmOptions::ATT_SYNTAX) { | |
628 | generated_asm.push_str(".att_syntax\n"); | |
629 | } | |
630 | ||
631 | // The actual inline asm | |
632 | for piece in self.template { | |
633 | match piece { | |
634 | InlineAsmTemplatePiece::String(s) => { | |
635 | generated_asm.push_str(s); | |
636 | } | |
637 | InlineAsmTemplatePiece::Placeholder { operand_idx, modifier, span: _ } => { | |
9ffffee4 FG |
638 | match self.operands[*operand_idx] { |
639 | CInlineAsmOperand::In { .. } | |
640 | | CInlineAsmOperand::Out { .. } | |
641 | | CInlineAsmOperand::InOut { .. } => { | |
642 | if self.options.contains(InlineAsmOptions::ATT_SYNTAX) { | |
643 | generated_asm.push('%'); | |
644 | } | |
645 | self.registers[*operand_idx] | |
646 | .unwrap() | |
647 | .emit(&mut generated_asm, self.arch, *modifier) | |
648 | .unwrap(); | |
649 | } | |
650 | CInlineAsmOperand::Const { ref value } => { | |
651 | generated_asm.push_str(value); | |
652 | } | |
653 | CInlineAsmOperand::Symbol { ref symbol } => generated_asm.push_str(symbol), | |
a2a8927a | 654 | } |
a2a8927a XL |
655 | } |
656 | } | |
657 | } | |
658 | generated_asm.push('\n'); | |
659 | ||
660 | if is_x86 && self.options.contains(InlineAsmOptions::ATT_SYNTAX) { | |
661 | generated_asm.push_str(".intel_syntax noprefix\n"); | |
662 | } | |
663 | ||
664 | if !self.options.contains(InlineAsmOptions::NORETURN) { | |
665 | // Read output registers | |
666 | for (reg, slot) in self | |
667 | .registers | |
668 | .iter() | |
669 | .zip(self.stack_slots_output.iter().copied()) | |
670 | .filter_map(|(r, s)| r.zip(s)) | |
671 | { | |
672 | Self::save_register(&mut generated_asm, self.arch, reg, slot); | |
673 | } | |
674 | ||
675 | // Restore clobbered registers | |
676 | for (reg, slot) in self | |
677 | .registers | |
678 | .iter() | |
679 | .zip(self.stack_slots_clobber.iter().copied()) | |
680 | .filter_map(|(r, s)| r.zip(s)) | |
681 | { | |
682 | Self::restore_register(&mut generated_asm, self.arch, reg, slot); | |
683 | } | |
684 | ||
685 | Self::epilogue(&mut generated_asm, self.arch); | |
686 | } else { | |
687 | Self::epilogue_noreturn(&mut generated_asm, self.arch); | |
688 | } | |
689 | ||
690 | if is_x86 { | |
691 | generated_asm.push_str(".att_syntax\n"); | |
692 | } | |
693 | writeln!(generated_asm, ".size {name}, .-{name}", name = asm_name).unwrap(); | |
694 | generated_asm.push_str(".text\n"); | |
695 | generated_asm.push_str("\n\n"); | |
696 | ||
697 | generated_asm | |
29967ef6 XL |
698 | } |
699 | ||
a2a8927a XL |
700 | fn prologue(generated_asm: &mut String, arch: InlineAsmArch) { |
701 | match arch { | |
702 | InlineAsmArch::X86 => { | |
703 | generated_asm.push_str(" push ebp\n"); | |
704 | generated_asm.push_str(" mov ebp,[esp+8]\n"); | |
705 | } | |
706 | InlineAsmArch::X86_64 => { | |
707 | generated_asm.push_str(" push rbp\n"); | |
708 | generated_asm.push_str(" mov rbp,rdi\n"); | |
709 | } | |
710 | InlineAsmArch::RiscV32 => { | |
711 | generated_asm.push_str(" addi sp, sp, -8\n"); | |
712 | generated_asm.push_str(" sw ra, 4(sp)\n"); | |
713 | generated_asm.push_str(" sw s0, 0(sp)\n"); | |
714 | generated_asm.push_str(" mv s0, a0\n"); | |
715 | } | |
716 | InlineAsmArch::RiscV64 => { | |
717 | generated_asm.push_str(" addi sp, sp, -16\n"); | |
718 | generated_asm.push_str(" sd ra, 8(sp)\n"); | |
719 | generated_asm.push_str(" sd s0, 0(sp)\n"); | |
720 | generated_asm.push_str(" mv s0, a0\n"); | |
721 | } | |
722 | _ => unimplemented!("prologue for {:?}", arch), | |
29967ef6 | 723 | } |
a2a8927a | 724 | } |
29967ef6 | 725 | |
a2a8927a XL |
726 | fn epilogue(generated_asm: &mut String, arch: InlineAsmArch) { |
727 | match arch { | |
728 | InlineAsmArch::X86 => { | |
729 | generated_asm.push_str(" pop ebp\n"); | |
730 | generated_asm.push_str(" ret\n"); | |
731 | } | |
732 | InlineAsmArch::X86_64 => { | |
733 | generated_asm.push_str(" pop rbp\n"); | |
734 | generated_asm.push_str(" ret\n"); | |
735 | } | |
736 | InlineAsmArch::RiscV32 => { | |
737 | generated_asm.push_str(" lw s0, 0(sp)\n"); | |
738 | generated_asm.push_str(" lw ra, 4(sp)\n"); | |
739 | generated_asm.push_str(" addi sp, sp, 8\n"); | |
740 | generated_asm.push_str(" ret\n"); | |
741 | } | |
742 | InlineAsmArch::RiscV64 => { | |
743 | generated_asm.push_str(" ld s0, 0(sp)\n"); | |
744 | generated_asm.push_str(" ld ra, 8(sp)\n"); | |
745 | generated_asm.push_str(" addi sp, sp, 16\n"); | |
746 | generated_asm.push_str(" ret\n"); | |
747 | } | |
748 | _ => unimplemented!("epilogue for {:?}", arch), | |
29967ef6 | 749 | } |
a2a8927a | 750 | } |
29967ef6 | 751 | |
a2a8927a XL |
752 | fn epilogue_noreturn(generated_asm: &mut String, arch: InlineAsmArch) { |
753 | match arch { | |
754 | InlineAsmArch::X86 | InlineAsmArch::X86_64 => { | |
755 | generated_asm.push_str(" ud2\n"); | |
756 | } | |
757 | InlineAsmArch::RiscV32 | InlineAsmArch::RiscV64 => { | |
758 | generated_asm.push_str(" ebreak\n"); | |
759 | } | |
760 | _ => unimplemented!("epilogue_noreturn for {:?}", arch), | |
761 | } | |
29967ef6 XL |
762 | } |
763 | ||
a2a8927a XL |
764 | fn save_register( |
765 | generated_asm: &mut String, | |
766 | arch: InlineAsmArch, | |
767 | reg: InlineAsmReg, | |
768 | offset: Size, | |
769 | ) { | |
770 | match arch { | |
771 | InlineAsmArch::X86 => { | |
772 | write!(generated_asm, " mov [ebp+0x{:x}], ", offset.bytes()).unwrap(); | |
773 | reg.emit(generated_asm, InlineAsmArch::X86, None).unwrap(); | |
774 | generated_asm.push('\n'); | |
775 | } | |
776 | InlineAsmArch::X86_64 => { | |
777 | write!(generated_asm, " mov [rbp+0x{:x}], ", offset.bytes()).unwrap(); | |
778 | reg.emit(generated_asm, InlineAsmArch::X86_64, None).unwrap(); | |
779 | generated_asm.push('\n'); | |
780 | } | |
781 | InlineAsmArch::RiscV32 => { | |
782 | generated_asm.push_str(" sw "); | |
783 | reg.emit(generated_asm, InlineAsmArch::RiscV32, None).unwrap(); | |
784 | writeln!(generated_asm, ", 0x{:x}(s0)", offset.bytes()).unwrap(); | |
785 | } | |
786 | InlineAsmArch::RiscV64 => { | |
787 | generated_asm.push_str(" sd "); | |
788 | reg.emit(generated_asm, InlineAsmArch::RiscV64, None).unwrap(); | |
789 | writeln!(generated_asm, ", 0x{:x}(s0)", offset.bytes()).unwrap(); | |
790 | } | |
791 | _ => unimplemented!("save_register for {:?}", arch), | |
792 | } | |
793 | } | |
29967ef6 | 794 | |
a2a8927a XL |
795 | fn restore_register( |
796 | generated_asm: &mut String, | |
797 | arch: InlineAsmArch, | |
798 | reg: InlineAsmReg, | |
799 | offset: Size, | |
800 | ) { | |
801 | match arch { | |
802 | InlineAsmArch::X86 => { | |
803 | generated_asm.push_str(" mov "); | |
804 | reg.emit(generated_asm, InlineAsmArch::X86, None).unwrap(); | |
805 | writeln!(generated_asm, ", [ebp+0x{:x}]", offset.bytes()).unwrap(); | |
806 | } | |
807 | InlineAsmArch::X86_64 => { | |
808 | generated_asm.push_str(" mov "); | |
809 | reg.emit(generated_asm, InlineAsmArch::X86_64, None).unwrap(); | |
810 | writeln!(generated_asm, ", [rbp+0x{:x}]", offset.bytes()).unwrap(); | |
811 | } | |
812 | InlineAsmArch::RiscV32 => { | |
813 | generated_asm.push_str(" lw "); | |
814 | reg.emit(generated_asm, InlineAsmArch::RiscV32, None).unwrap(); | |
815 | writeln!(generated_asm, ", 0x{:x}(s0)", offset.bytes()).unwrap(); | |
816 | } | |
817 | InlineAsmArch::RiscV64 => { | |
818 | generated_asm.push_str(" ld "); | |
819 | reg.emit(generated_asm, InlineAsmArch::RiscV64, None).unwrap(); | |
820 | writeln!(generated_asm, ", 0x{:x}(s0)", offset.bytes()).unwrap(); | |
821 | } | |
822 | _ => unimplemented!("restore_register for {:?}", arch), | |
823 | } | |
824 | } | |
29967ef6 XL |
825 | } |
826 | ||
827 | fn call_inline_asm<'tcx>( | |
6a06907d | 828 | fx: &mut FunctionCx<'_, '_, 'tcx>, |
29967ef6 XL |
829 | asm_name: &str, |
830 | slot_size: Size, | |
a2a8927a XL |
831 | inputs: Vec<(Size, Value)>, |
832 | outputs: Vec<(Size, CPlace<'tcx>)>, | |
29967ef6 | 833 | ) { |
f2b60f7d | 834 | let stack_slot = fx.bcx.func.create_sized_stack_slot(StackSlotData { |
29967ef6 | 835 | kind: StackSlotKind::ExplicitSlot, |
29967ef6 XL |
836 | size: u32::try_from(slot_size.bytes()).unwrap(), |
837 | }); | |
cdc7bbd5 XL |
838 | if fx.clif_comments.enabled() { |
839 | fx.add_comment(stack_slot, "inline asm scratch slot"); | |
840 | } | |
29967ef6 XL |
841 | |
842 | let inline_asm_func = fx | |
29967ef6 XL |
843 | .module |
844 | .declare_function( | |
845 | asm_name, | |
846 | Linkage::Import, | |
847 | &Signature { | |
848 | call_conv: CallConv::SystemV, | |
849 | params: vec![AbiParam::new(fx.pointer_type)], | |
850 | returns: vec![], | |
851 | }, | |
852 | ) | |
853 | .unwrap(); | |
17df50a5 | 854 | let inline_asm_func = fx.module.declare_func_in_func(inline_asm_func, &mut fx.bcx.func); |
cdc7bbd5 XL |
855 | if fx.clif_comments.enabled() { |
856 | fx.add_comment(inline_asm_func, asm_name); | |
857 | } | |
29967ef6 | 858 | |
a2a8927a | 859 | for (offset, value) in inputs { |
6a06907d | 860 | fx.bcx.ins().stack_store(value, stack_slot, i32::try_from(offset.bytes()).unwrap()); |
29967ef6 XL |
861 | } |
862 | ||
863 | let stack_slot_addr = fx.bcx.ins().stack_addr(fx.pointer_type, stack_slot, 0); | |
864 | fx.bcx.ins().call(inline_asm_func, &[stack_slot_addr]); | |
865 | ||
a2a8927a | 866 | for (offset, place) in outputs { |
29967ef6 | 867 | let ty = fx.clif_type(place.layout().ty).unwrap(); |
6a06907d | 868 | let value = fx.bcx.ins().stack_load(ty, stack_slot, i32::try_from(offset.bytes()).unwrap()); |
29967ef6 XL |
869 | place.write_cvalue(fx, CValue::by_val(value, place.layout())); |
870 | } | |
871 | } |