]>
Commit | Line | Data |
---|---|---|
29967ef6 XL |
1 | //! Codegen of [`asm!`] invocations. |
2 | ||
3 | use crate::prelude::*; | |
4 | ||
5 | use std::fmt::Write; | |
6 | ||
7 | use rustc_ast::ast::{InlineAsmOptions, InlineAsmTemplatePiece}; | |
8 | use rustc_middle::mir::InlineAsmOperand; | |
9 | use rustc_target::asm::*; | |
10 | ||
11 | pub(crate) fn codegen_inline_asm<'tcx>( | |
6a06907d | 12 | fx: &mut FunctionCx<'_, '_, 'tcx>, |
29967ef6 XL |
13 | _span: Span, |
14 | template: &[InlineAsmTemplatePiece], | |
15 | operands: &[InlineAsmOperand<'tcx>], | |
16 | options: InlineAsmOptions, | |
17 | ) { | |
18 | // FIXME add .eh_frame unwind info directives | |
19 | ||
20 | if template.is_empty() { | |
21 | // Black box | |
22 | return; | |
23 | } | |
24 | ||
25 | let mut slot_size = Size::from_bytes(0); | |
26 | let mut clobbered_regs = Vec::new(); | |
27 | let mut inputs = Vec::new(); | |
28 | let mut outputs = Vec::new(); | |
29 | ||
30 | let mut new_slot = |reg_class: InlineAsmRegClass| { | |
31 | let reg_size = reg_class | |
32 | .supported_types(InlineAsmArch::X86_64) | |
33 | .iter() | |
34 | .map(|(ty, _)| ty.size()) | |
35 | .max() | |
36 | .unwrap(); | |
37 | let align = rustc_target::abi::Align::from_bytes(reg_size.bytes()).unwrap(); | |
38 | slot_size = slot_size.align_to(align); | |
39 | let offset = slot_size; | |
40 | slot_size += reg_size; | |
41 | offset | |
42 | }; | |
43 | ||
44 | // FIXME overlap input and output slots to save stack space | |
45 | for operand in operands { | |
46 | match *operand { | |
47 | InlineAsmOperand::In { reg, ref value } => { | |
48 | let reg = expect_reg(reg); | |
49 | clobbered_regs.push((reg, new_slot(reg.reg_class()))); | |
50 | inputs.push(( | |
51 | reg, | |
52 | new_slot(reg.reg_class()), | |
53 | crate::base::codegen_operand(fx, value).load_scalar(fx), | |
54 | )); | |
55 | } | |
6a06907d | 56 | InlineAsmOperand::Out { reg, late: _, place } => { |
29967ef6 XL |
57 | let reg = expect_reg(reg); |
58 | clobbered_regs.push((reg, new_slot(reg.reg_class()))); | |
59 | if let Some(place) = place { | |
60 | outputs.push(( | |
61 | reg, | |
62 | new_slot(reg.reg_class()), | |
63 | crate::base::codegen_place(fx, place), | |
64 | )); | |
65 | } | |
66 | } | |
6a06907d | 67 | InlineAsmOperand::InOut { reg, late: _, ref in_value, out_place } => { |
29967ef6 XL |
68 | let reg = expect_reg(reg); |
69 | clobbered_regs.push((reg, new_slot(reg.reg_class()))); | |
70 | inputs.push(( | |
71 | reg, | |
72 | new_slot(reg.reg_class()), | |
73 | crate::base::codegen_operand(fx, in_value).load_scalar(fx), | |
74 | )); | |
75 | if let Some(out_place) = out_place { | |
76 | outputs.push(( | |
77 | reg, | |
78 | new_slot(reg.reg_class()), | |
79 | crate::base::codegen_place(fx, out_place), | |
80 | )); | |
81 | } | |
82 | } | |
83 | InlineAsmOperand::Const { value: _ } => todo!(), | |
84 | InlineAsmOperand::SymFn { value: _ } => todo!(), | |
85 | InlineAsmOperand::SymStatic { def_id: _ } => todo!(), | |
86 | } | |
87 | } | |
88 | ||
89 | let inline_asm_index = fx.inline_asm_index; | |
90 | fx.inline_asm_index += 1; | |
6a06907d XL |
91 | let asm_name = |
92 | format!("{}__inline_asm_{}", fx.tcx.symbol_name(fx.instance).name, inline_asm_index); | |
29967ef6 XL |
93 | |
94 | let generated_asm = generate_asm_wrapper( | |
95 | &asm_name, | |
96 | InlineAsmArch::X86_64, | |
97 | options, | |
98 | template, | |
99 | clobbered_regs, | |
100 | &inputs, | |
101 | &outputs, | |
102 | ); | |
103 | fx.cx.global_asm.push_str(&generated_asm); | |
104 | ||
105 | call_inline_asm(fx, &asm_name, slot_size, inputs, outputs); | |
106 | } | |
107 | ||
108 | fn generate_asm_wrapper( | |
109 | asm_name: &str, | |
110 | arch: InlineAsmArch, | |
111 | options: InlineAsmOptions, | |
112 | template: &[InlineAsmTemplatePiece], | |
113 | clobbered_regs: Vec<(InlineAsmReg, Size)>, | |
114 | inputs: &[(InlineAsmReg, Size, Value)], | |
115 | outputs: &[(InlineAsmReg, Size, CPlace<'_>)], | |
116 | ) -> String { | |
117 | let mut generated_asm = String::new(); | |
118 | writeln!(generated_asm, ".globl {}", asm_name).unwrap(); | |
119 | writeln!(generated_asm, ".type {},@function", asm_name).unwrap(); | |
6a06907d | 120 | writeln!(generated_asm, ".section .text.{},\"ax\",@progbits", asm_name).unwrap(); |
29967ef6 XL |
121 | writeln!(generated_asm, "{}:", asm_name).unwrap(); |
122 | ||
123 | generated_asm.push_str(".intel_syntax noprefix\n"); | |
124 | generated_asm.push_str(" push rbp\n"); | |
125 | generated_asm.push_str(" mov rbp,rdi\n"); | |
126 | ||
127 | // Save clobbered registers | |
128 | if !options.contains(InlineAsmOptions::NORETURN) { | |
129 | // FIXME skip registers saved by the calling convention | |
130 | for &(reg, offset) in &clobbered_regs { | |
131 | save_register(&mut generated_asm, arch, reg, offset); | |
132 | } | |
133 | } | |
134 | ||
135 | // Write input registers | |
136 | for &(reg, offset, _value) in inputs { | |
137 | restore_register(&mut generated_asm, arch, reg, offset); | |
138 | } | |
139 | ||
140 | if options.contains(InlineAsmOptions::ATT_SYNTAX) { | |
141 | generated_asm.push_str(".att_syntax\n"); | |
142 | } | |
143 | ||
144 | // The actual inline asm | |
145 | for piece in template { | |
146 | match piece { | |
147 | InlineAsmTemplatePiece::String(s) => { | |
148 | generated_asm.push_str(s); | |
149 | } | |
6a06907d | 150 | InlineAsmTemplatePiece::Placeholder { operand_idx: _, modifier: _, span: _ } => todo!(), |
29967ef6 XL |
151 | } |
152 | } | |
153 | generated_asm.push('\n'); | |
154 | ||
155 | if options.contains(InlineAsmOptions::ATT_SYNTAX) { | |
156 | generated_asm.push_str(".intel_syntax noprefix\n"); | |
157 | } | |
158 | ||
159 | if !options.contains(InlineAsmOptions::NORETURN) { | |
160 | // Read output registers | |
161 | for &(reg, offset, _place) in outputs { | |
162 | save_register(&mut generated_asm, arch, reg, offset); | |
163 | } | |
164 | ||
165 | // Restore clobbered registers | |
166 | for &(reg, offset) in clobbered_regs.iter().rev() { | |
167 | restore_register(&mut generated_asm, arch, reg, offset); | |
168 | } | |
169 | ||
170 | generated_asm.push_str(" pop rbp\n"); | |
171 | generated_asm.push_str(" ret\n"); | |
172 | } else { | |
173 | generated_asm.push_str(" ud2\n"); | |
174 | } | |
175 | ||
176 | generated_asm.push_str(".att_syntax\n"); | |
177 | writeln!(generated_asm, ".size {name}, .-{name}", name = asm_name).unwrap(); | |
178 | generated_asm.push_str(".text\n"); | |
179 | generated_asm.push_str("\n\n"); | |
180 | ||
181 | generated_asm | |
182 | } | |
183 | ||
184 | fn call_inline_asm<'tcx>( | |
6a06907d | 185 | fx: &mut FunctionCx<'_, '_, 'tcx>, |
29967ef6 XL |
186 | asm_name: &str, |
187 | slot_size: Size, | |
188 | inputs: Vec<(InlineAsmReg, Size, Value)>, | |
189 | outputs: Vec<(InlineAsmReg, Size, CPlace<'tcx>)>, | |
190 | ) { | |
191 | let stack_slot = fx.bcx.func.create_stack_slot(StackSlotData { | |
192 | kind: StackSlotKind::ExplicitSlot, | |
193 | offset: None, | |
194 | size: u32::try_from(slot_size.bytes()).unwrap(), | |
195 | }); | |
196 | #[cfg(debug_assertions)] | |
197 | fx.add_comment(stack_slot, "inline asm scratch slot"); | |
198 | ||
199 | let inline_asm_func = fx | |
200 | .cx | |
201 | .module | |
202 | .declare_function( | |
203 | asm_name, | |
204 | Linkage::Import, | |
205 | &Signature { | |
206 | call_conv: CallConv::SystemV, | |
207 | params: vec![AbiParam::new(fx.pointer_type)], | |
208 | returns: vec![], | |
209 | }, | |
210 | ) | |
211 | .unwrap(); | |
6a06907d | 212 | let inline_asm_func = fx.cx.module.declare_func_in_func(inline_asm_func, &mut fx.bcx.func); |
29967ef6 XL |
213 | #[cfg(debug_assertions)] |
214 | fx.add_comment(inline_asm_func, asm_name); | |
215 | ||
216 | for (_reg, offset, value) in inputs { | |
6a06907d | 217 | fx.bcx.ins().stack_store(value, stack_slot, i32::try_from(offset.bytes()).unwrap()); |
29967ef6 XL |
218 | } |
219 | ||
220 | let stack_slot_addr = fx.bcx.ins().stack_addr(fx.pointer_type, stack_slot, 0); | |
221 | fx.bcx.ins().call(inline_asm_func, &[stack_slot_addr]); | |
222 | ||
223 | for (_reg, offset, place) in outputs { | |
224 | let ty = fx.clif_type(place.layout().ty).unwrap(); | |
6a06907d | 225 | let value = fx.bcx.ins().stack_load(ty, stack_slot, i32::try_from(offset.bytes()).unwrap()); |
29967ef6 XL |
226 | place.write_cvalue(fx, CValue::by_val(value, place.layout())); |
227 | } | |
228 | } | |
229 | ||
230 | fn expect_reg(reg_or_class: InlineAsmRegOrRegClass) -> InlineAsmReg { | |
231 | match reg_or_class { | |
232 | InlineAsmRegOrRegClass::Reg(reg) => reg, | |
233 | InlineAsmRegOrRegClass::RegClass(class) => unimplemented!("{:?}", class), | |
234 | } | |
235 | } | |
236 | ||
237 | fn save_register(generated_asm: &mut String, arch: InlineAsmArch, reg: InlineAsmReg, offset: Size) { | |
238 | match arch { | |
239 | InlineAsmArch::X86_64 => { | |
240 | write!(generated_asm, " mov [rbp+0x{:x}], ", offset.bytes()).unwrap(); | |
6a06907d | 241 | reg.emit(generated_asm, InlineAsmArch::X86_64, None).unwrap(); |
29967ef6 XL |
242 | generated_asm.push('\n'); |
243 | } | |
244 | _ => unimplemented!("save_register for {:?}", arch), | |
245 | } | |
246 | } | |
247 | ||
248 | fn restore_register( | |
249 | generated_asm: &mut String, | |
250 | arch: InlineAsmArch, | |
251 | reg: InlineAsmReg, | |
252 | offset: Size, | |
253 | ) { | |
254 | match arch { | |
255 | InlineAsmArch::X86_64 => { | |
256 | generated_asm.push_str(" mov "); | |
6a06907d | 257 | reg.emit(generated_asm, InlineAsmArch::X86_64, None).unwrap(); |
29967ef6 XL |
258 | writeln!(generated_asm, ", [rbp+0x{:x}]", offset.bytes()).unwrap(); |
259 | } | |
260 | _ => unimplemented!("restore_register for {:?}", arch), | |
261 | } | |
262 | } |