]>
Commit | Line | Data |
---|---|---|
c295e0f8 XL |
1 | use gccjit::{LValue, RValue, ToRValue, Type}; |
2 | use rustc_ast::ast::{InlineAsmOptions, InlineAsmTemplatePiece}; | |
3 | use rustc_codegen_ssa::mir::operand::OperandValue; | |
4 | use rustc_codegen_ssa::mir::place::PlaceRef; | |
5 | use rustc_codegen_ssa::traits::{AsmBuilderMethods, AsmMethods, BaseTypeMethods, BuilderMethods, GlobalAsmOperandRef, InlineAsmOperandRef}; | |
6 | ||
c295e0f8 | 7 | use rustc_middle::{bug, ty::Instance}; |
5099ac24 | 8 | use rustc_span::Span; |
c295e0f8 XL |
9 | use rustc_target::asm::*; |
10 | ||
11 | use std::borrow::Cow; | |
12 | ||
13 | use crate::builder::Builder; | |
14 | use crate::context::CodegenCx; | |
2b03887a | 15 | use crate::errors::UnwindingInlineAsm; |
c295e0f8 | 16 | use crate::type_of::LayoutGccExt; |
923072b8 | 17 | use crate::callee::get_fn; |
c295e0f8 XL |
18 | |
19 | ||
20 | // Rust asm! and GCC Extended Asm semantics differ substantially. | |
21 | // | |
a2a8927a XL |
22 | // 1. Rust asm operands go along as one list of operands. Operands themselves indicate |
23 | // if they're "in" or "out". "In" and "out" operands can interleave. One operand can be | |
c295e0f8 XL |
24 | // both "in" and "out" (`inout(reg)`). |
25 | // | |
a2a8927a XL |
26 | // GCC asm has two different lists for "in" and "out" operands. In terms of gccjit, |
27 | // this means that all "out" operands must go before "in" operands. "In" and "out" operands | |
c295e0f8 XL |
28 | // cannot interleave. |
29 | // | |
a2a8927a | 30 | // 2. Operand lists in both Rust and GCC are indexed. Index starts from 0. Indexes are important |
c295e0f8 XL |
31 | // because the asm template refers to operands by index. |
32 | // | |
33 | // Mapping from Rust to GCC index would be 1-1 if it wasn't for... | |
34 | // | |
a2a8927a XL |
35 | // 3. Clobbers. GCC has a separate list of clobbers, and clobbers don't have indexes. |
36 | // Contrary, Rust expresses clobbers through "out" operands that aren't tied to | |
c295e0f8 XL |
37 | // a variable (`_`), and such "clobbers" do have index. |
38 | // | |
a2a8927a XL |
39 | // 4. Furthermore, GCC Extended Asm does not support explicit register constraints |
40 | // (like `out("eax")`) directly, offering so-called "local register variables" | |
41 | // as a workaround. These variables need to be declared and initialized *before* | |
42 | // the Extended Asm block but *after* normal local variables | |
c295e0f8 XL |
43 | // (see comment in `codegen_inline_asm` for explanation). |
44 | // | |
a2a8927a | 45 | // With that in mind, let's see how we translate Rust syntax to GCC |
c295e0f8 XL |
46 | // (from now on, `CC` stands for "constraint code"): |
47 | // | |
48 | // * `out(reg_class) var` -> translated to output operand: `"=CC"(var)` | |
49 | // * `inout(reg_class) var` -> translated to output operand: `"+CC"(var)` | |
50 | // * `in(reg_class) var` -> translated to input operand: `"CC"(var)` | |
51 | // | |
52 | // * `out(reg_class) _` -> translated to one `=r(tmp)`, where "tmp" is a temporary unused variable | |
53 | // | |
54 | // * `out("explicit register") _` -> not translated to any operands, register is simply added to clobbers list | |
55 | // | |
a2a8927a | 56 | // * `inout(reg_class) in_var => out_var` -> translated to two operands: |
c295e0f8 | 57 | // output: `"=CC"(in_var)` |
a2a8927a | 58 | // input: `"num"(out_var)` where num is the GCC index |
c295e0f8 XL |
59 | // of the corresponding output operand |
60 | // | |
a2a8927a | 61 | // * `inout(reg_class) in_var => _` -> same as `inout(reg_class) in_var => tmp`, |
c295e0f8 XL |
62 | // where "tmp" is a temporary unused variable |
63 | // | |
a2a8927a XL |
64 | // * `out/in/inout("explicit register") var` -> translated to one or two operands as described above |
65 | // with `"r"(var)` constraint, | |
c295e0f8 | 66 | // and one register variable assigned to the desired register. |
c295e0f8 XL |
67 | |
68 | const ATT_SYNTAX_INS: &str = ".att_syntax noprefix\n\t"; | |
69 | const INTEL_SYNTAX_INS: &str = "\n\t.intel_syntax noprefix"; | |
70 | ||
71 | ||
72 | struct AsmOutOperand<'a, 'tcx, 'gcc> { | |
73 | rust_idx: usize, | |
74 | constraint: &'a str, | |
75 | late: bool, | |
76 | readwrite: bool, | |
77 | ||
78 | tmp_var: LValue<'gcc>, | |
79 | out_place: Option<PlaceRef<'tcx, RValue<'gcc>>> | |
80 | } | |
81 | ||
82 | struct AsmInOperand<'a, 'tcx> { | |
83 | rust_idx: usize, | |
84 | constraint: Cow<'a, str>, | |
85 | val: RValue<'tcx> | |
86 | } | |
87 | ||
88 | impl AsmOutOperand<'_, '_, '_> { | |
89 | fn to_constraint(&self) -> String { | |
90 | let mut res = String::with_capacity(self.constraint.len() + self.late as usize + 1); | |
91 | ||
92 | let sign = if self.readwrite { '+' } else { '=' }; | |
93 | res.push(sign); | |
94 | if !self.late { | |
95 | res.push('&'); | |
96 | } | |
97 | ||
98 | res.push_str(&self.constraint); | |
99 | res | |
100 | } | |
101 | } | |
102 | ||
103 | enum ConstraintOrRegister { | |
104 | Constraint(&'static str), | |
105 | Register(&'static str) | |
106 | } | |
107 | ||
108 | ||
109 | impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> { | |
a2a8927a XL |
110 | fn codegen_inline_asm(&mut self, template: &[InlineAsmTemplatePiece], rust_operands: &[InlineAsmOperandRef<'tcx, Self>], options: InlineAsmOptions, span: &[Span], _instance: Instance<'_>, _dest_catch_funclet: Option<(Self::BasicBlock, Self::BasicBlock, Option<&Self::Funclet>)>) { |
111 | if options.contains(InlineAsmOptions::MAY_UNWIND) { | |
112 | self.sess() | |
2b03887a | 113 | .create_err(UnwindingInlineAsm { span: span[0] }) |
a2a8927a XL |
114 | .emit(); |
115 | return; | |
116 | } | |
117 | ||
c295e0f8 XL |
118 | let asm_arch = self.tcx.sess.asm_arch.unwrap(); |
119 | let is_x86 = matches!(asm_arch, InlineAsmArch::X86 | InlineAsmArch::X86_64); | |
120 | let att_dialect = is_x86 && options.contains(InlineAsmOptions::ATT_SYNTAX); | |
c295e0f8 | 121 | |
a2a8927a | 122 | // GCC index of an output operand equals its position in the array |
c295e0f8 XL |
123 | let mut outputs = vec![]; |
124 | ||
125 | // GCC index of an input operand equals its position in the array | |
126 | // added to `outputs.len()` | |
127 | let mut inputs = vec![]; | |
128 | ||
129 | // Clobbers collected from `out("explicit register") _` and `inout("expl_reg") var => _` | |
130 | let mut clobbers = vec![]; | |
131 | ||
132 | // We're trying to preallocate space for the template | |
133 | let mut constants_len = 0; | |
134 | ||
135 | // There are rules we must adhere to if we want GCC to do the right thing: | |
a2a8927a | 136 | // |
c295e0f8 | 137 | // * Every local variable that the asm block uses as an output must be declared *before* |
a2a8927a | 138 | // the asm block. |
c295e0f8 XL |
139 | // * There must be no instructions whatsoever between the register variables and the asm. |
140 | // | |
141 | // Therefore, the backend must generate the instructions strictly in this order: | |
142 | // | |
143 | // 1. Output variables. | |
144 | // 2. Register variables. | |
145 | // 3. The asm block. | |
146 | // | |
147 | // We also must make sure that no input operands are emitted before output operands. | |
148 | // | |
149 | // This is why we work in passes, first emitting local vars, then local register vars. | |
a2a8927a | 150 | // Also, we don't emit any asm operands immediately; we save them to |
c295e0f8 XL |
151 | // the one of the buffers to be emitted later. |
152 | ||
153 | // 1. Normal variables (and saving operands to buffers). | |
154 | for (rust_idx, op) in rust_operands.iter().enumerate() { | |
155 | match *op { | |
156 | InlineAsmOperandRef::Out { reg, late, place } => { | |
157 | use ConstraintOrRegister::*; | |
158 | ||
159 | let (constraint, ty) = match (reg_to_gcc(reg), place) { | |
160 | (Constraint(constraint), Some(place)) => (constraint, place.layout.gcc_type(self.cx, false)), | |
161 | // When `reg` is a class and not an explicit register but the out place is not specified, | |
162 | // we need to create an unused output variable to assign the output to. This var | |
a2a8927a | 163 | // needs to be of a type that's "compatible" with the register class, but specific type |
c295e0f8 XL |
164 | // doesn't matter. |
165 | (Constraint(constraint), None) => (constraint, dummy_output_type(self.cx, reg.reg_class())), | |
166 | (Register(_), Some(_)) => { | |
167 | // left for the next pass | |
168 | continue | |
169 | }, | |
170 | (Register(reg_name), None) => { | |
171 | // `clobber_abi` can add lots of clobbers that are not supported by the target, | |
172 | // such as AVX-512 registers, so we just ignore unsupported registers | |
173 | let is_target_supported = reg.reg_class().supported_types(asm_arch).iter() | |
174 | .any(|&(_, feature)| { | |
175 | if let Some(feature) = feature { | |
5099ac24 | 176 | self.tcx.sess.target_features.contains(&feature) |
c295e0f8 XL |
177 | } else { |
178 | true // Register class is unconditionally supported | |
179 | } | |
180 | }); | |
181 | ||
182 | if is_target_supported && !clobbers.contains(®_name) { | |
183 | clobbers.push(reg_name); | |
184 | } | |
185 | continue | |
186 | } | |
187 | }; | |
188 | ||
189 | let tmp_var = self.current_func().new_local(None, ty, "output_register"); | |
190 | outputs.push(AsmOutOperand { | |
a2a8927a | 191 | constraint, |
c295e0f8 XL |
192 | rust_idx, |
193 | late, | |
194 | readwrite: false, | |
195 | tmp_var, | |
196 | out_place: place | |
197 | }); | |
198 | } | |
199 | ||
200 | InlineAsmOperandRef::In { reg, value } => { | |
201 | if let ConstraintOrRegister::Constraint(constraint) = reg_to_gcc(reg) { | |
a2a8927a XL |
202 | inputs.push(AsmInOperand { |
203 | constraint: Cow::Borrowed(constraint), | |
204 | rust_idx, | |
c295e0f8 XL |
205 | val: value.immediate() |
206 | }); | |
a2a8927a | 207 | } |
c295e0f8 XL |
208 | else { |
209 | // left for the next pass | |
210 | continue | |
211 | } | |
212 | } | |
213 | ||
214 | InlineAsmOperandRef::InOut { reg, late, in_value, out_place } => { | |
215 | let constraint = if let ConstraintOrRegister::Constraint(constraint) = reg_to_gcc(reg) { | |
216 | constraint | |
a2a8927a | 217 | } |
c295e0f8 XL |
218 | else { |
219 | // left for the next pass | |
220 | continue | |
221 | }; | |
222 | ||
223 | // Rustc frontend guarantees that input and output types are "compatible", | |
224 | // so we can just use input var's type for the output variable. | |
225 | // | |
a2a8927a XL |
226 | // This decision is also backed by the fact that LLVM needs in and out |
227 | // values to be of *exactly the same type*, not just "compatible". | |
c295e0f8 XL |
228 | // I'm not sure if GCC is so picky too, but better safe than sorry. |
229 | let ty = in_value.layout.gcc_type(self.cx, false); | |
230 | let tmp_var = self.current_func().new_local(None, ty, "output_register"); | |
231 | ||
232 | // If the out_place is None (i.e `inout(reg) _` syntax was used), we translate | |
a2a8927a | 233 | // it to one "readwrite (+) output variable", otherwise we translate it to two |
c295e0f8 XL |
234 | // "out and tied in" vars as described above. |
235 | let readwrite = out_place.is_none(); | |
236 | outputs.push(AsmOutOperand { | |
a2a8927a | 237 | constraint, |
c295e0f8 XL |
238 | rust_idx, |
239 | late, | |
240 | readwrite, | |
a2a8927a | 241 | tmp_var, |
c295e0f8 XL |
242 | out_place, |
243 | }); | |
244 | ||
245 | if !readwrite { | |
246 | let out_gcc_idx = outputs.len() - 1; | |
247 | let constraint = Cow::Owned(out_gcc_idx.to_string()); | |
248 | ||
249 | inputs.push(AsmInOperand { | |
a2a8927a XL |
250 | constraint, |
251 | rust_idx, | |
c295e0f8 XL |
252 | val: in_value.immediate() |
253 | }); | |
254 | } | |
255 | } | |
256 | ||
257 | InlineAsmOperandRef::Const { ref string } => { | |
258 | constants_len += string.len() + att_dialect as usize; | |
259 | } | |
260 | ||
261 | InlineAsmOperandRef::SymFn { instance } => { | |
04454e1e FG |
262 | // TODO(@Amanieu): Additional mangling is needed on |
263 | // some targets to add a leading underscore (Mach-O) | |
264 | // or byte count suffixes (x86 Windows). | |
c295e0f8 XL |
265 | constants_len += self.tcx.symbol_name(instance).name.len(); |
266 | } | |
267 | InlineAsmOperandRef::SymStatic { def_id } => { | |
04454e1e FG |
268 | // TODO(@Amanieu): Additional mangling is needed on |
269 | // some targets to add a leading underscore (Mach-O). | |
c295e0f8 XL |
270 | constants_len += self.tcx.symbol_name(Instance::mono(self.tcx, def_id)).name.len(); |
271 | } | |
272 | } | |
273 | } | |
274 | ||
275 | // 2. Register variables. | |
276 | for (rust_idx, op) in rust_operands.iter().enumerate() { | |
277 | match *op { | |
278 | // `out("explicit register") var` | |
279 | InlineAsmOperandRef::Out { reg, late, place } => { | |
280 | if let ConstraintOrRegister::Register(reg_name) = reg_to_gcc(reg) { | |
281 | let out_place = if let Some(place) = place { | |
282 | place | |
a2a8927a | 283 | } |
c295e0f8 XL |
284 | else { |
285 | // processed in the previous pass | |
286 | continue | |
287 | }; | |
288 | ||
289 | let ty = out_place.layout.gcc_type(self.cx, false); | |
290 | let tmp_var = self.current_func().new_local(None, ty, "output_register"); | |
291 | tmp_var.set_register_name(reg_name); | |
292 | ||
293 | outputs.push(AsmOutOperand { | |
a2a8927a | 294 | constraint: "r".into(), |
c295e0f8 XL |
295 | rust_idx, |
296 | late, | |
297 | readwrite: false, | |
298 | tmp_var, | |
299 | out_place: Some(out_place) | |
300 | }); | |
301 | } | |
302 | ||
303 | // processed in the previous pass | |
304 | } | |
305 | ||
306 | // `in("explicit register") var` | |
307 | InlineAsmOperandRef::In { reg, value } => { | |
308 | if let ConstraintOrRegister::Register(reg_name) = reg_to_gcc(reg) { | |
309 | let ty = value.layout.gcc_type(self.cx, false); | |
310 | let reg_var = self.current_func().new_local(None, ty, "input_register"); | |
311 | reg_var.set_register_name(reg_name); | |
312 | self.llbb().add_assignment(None, reg_var, value.immediate()); | |
313 | ||
a2a8927a XL |
314 | inputs.push(AsmInOperand { |
315 | constraint: "r".into(), | |
316 | rust_idx, | |
c295e0f8 XL |
317 | val: reg_var.to_rvalue() |
318 | }); | |
319 | } | |
320 | ||
321 | // processed in the previous pass | |
322 | } | |
323 | ||
324 | // `inout("explicit register") in_var => out_var` | |
325 | InlineAsmOperandRef::InOut { reg, late, in_value, out_place } => { | |
326 | if let ConstraintOrRegister::Register(reg_name) = reg_to_gcc(reg) { | |
c295e0f8 XL |
327 | // See explanation in the first pass. |
328 | let ty = in_value.layout.gcc_type(self.cx, false); | |
329 | let tmp_var = self.current_func().new_local(None, ty, "output_register"); | |
330 | tmp_var.set_register_name(reg_name); | |
331 | ||
332 | outputs.push(AsmOutOperand { | |
a2a8927a | 333 | constraint: "r".into(), |
c295e0f8 XL |
334 | rust_idx, |
335 | late, | |
336 | readwrite: false, | |
337 | tmp_var, | |
a2a8927a | 338 | out_place, |
c295e0f8 XL |
339 | }); |
340 | ||
341 | let constraint = Cow::Owned((outputs.len() - 1).to_string()); | |
a2a8927a XL |
342 | inputs.push(AsmInOperand { |
343 | constraint, | |
c295e0f8 XL |
344 | rust_idx, |
345 | val: in_value.immediate() | |
346 | }); | |
347 | } | |
348 | ||
349 | // processed in the previous pass | |
350 | } | |
351 | ||
923072b8 FG |
352 | InlineAsmOperandRef::SymFn { instance } => { |
353 | inputs.push(AsmInOperand { | |
354 | constraint: "X".into(), | |
355 | rust_idx, | |
356 | val: self.cx.rvalue_as_function(get_fn(self.cx, instance)) | |
357 | .get_address(None), | |
358 | }); | |
359 | } | |
360 | ||
361 | InlineAsmOperandRef::SymStatic { def_id } => { | |
362 | inputs.push(AsmInOperand { | |
363 | constraint: "X".into(), | |
364 | rust_idx, | |
365 | val: self.cx.get_static(def_id).get_address(None), | |
366 | }); | |
367 | } | |
368 | ||
369 | InlineAsmOperandRef::Const { .. } => { | |
c295e0f8 XL |
370 | // processed in the previous pass |
371 | } | |
372 | } | |
373 | } | |
374 | ||
375 | // 3. Build the template string | |
376 | ||
377 | let mut template_str = String::with_capacity(estimate_template_length(template, constants_len, att_dialect)); | |
923072b8 | 378 | if att_dialect { |
c295e0f8 XL |
379 | template_str.push_str(ATT_SYNTAX_INS); |
380 | } | |
381 | ||
382 | for piece in template { | |
383 | match *piece { | |
384 | InlineAsmTemplatePiece::String(ref string) => { | |
385 | // TODO(@Commeownist): switch to `Iterator::intersperse` once it's stable | |
386 | let mut iter = string.split('%'); | |
387 | if let Some(s) = iter.next() { | |
388 | template_str.push_str(s); | |
389 | } | |
390 | ||
391 | for s in iter { | |
392 | template_str.push_str("%%"); | |
393 | template_str.push_str(s); | |
394 | } | |
395 | } | |
396 | InlineAsmTemplatePiece::Placeholder { operand_idx, modifier, span: _ } => { | |
397 | let mut push_to_template = |modifier, gcc_idx| { | |
398 | use std::fmt::Write; | |
399 | ||
400 | template_str.push('%'); | |
401 | if let Some(modifier) = modifier { | |
402 | template_str.push(modifier); | |
403 | } | |
404 | write!(template_str, "{}", gcc_idx).expect("pushing to string failed"); | |
405 | }; | |
406 | ||
407 | match rust_operands[operand_idx] { | |
408 | InlineAsmOperandRef::Out { reg, .. } => { | |
409 | let modifier = modifier_to_gcc(asm_arch, reg.reg_class(), modifier); | |
410 | let gcc_index = outputs.iter() | |
411 | .position(|op| operand_idx == op.rust_idx) | |
412 | .expect("wrong rust index"); | |
413 | push_to_template(modifier, gcc_index); | |
414 | } | |
415 | ||
416 | InlineAsmOperandRef::In { reg, .. } => { | |
417 | let modifier = modifier_to_gcc(asm_arch, reg.reg_class(), modifier); | |
418 | let in_gcc_index = inputs.iter() | |
419 | .position(|op| operand_idx == op.rust_idx) | |
420 | .expect("wrong rust index"); | |
421 | let gcc_index = in_gcc_index + outputs.len(); | |
422 | push_to_template(modifier, gcc_index); | |
423 | } | |
424 | ||
425 | InlineAsmOperandRef::InOut { reg, .. } => { | |
426 | let modifier = modifier_to_gcc(asm_arch, reg.reg_class(), modifier); | |
427 | ||
428 | // The input register is tied to the output, so we can just use the index of the output register | |
429 | let gcc_index = outputs.iter() | |
430 | .position(|op| operand_idx == op.rust_idx) | |
431 | .expect("wrong rust index"); | |
432 | push_to_template(modifier, gcc_index); | |
433 | } | |
434 | ||
435 | InlineAsmOperandRef::SymFn { instance } => { | |
04454e1e FG |
436 | // TODO(@Amanieu): Additional mangling is needed on |
437 | // some targets to add a leading underscore (Mach-O) | |
438 | // or byte count suffixes (x86 Windows). | |
c295e0f8 XL |
439 | let name = self.tcx.symbol_name(instance).name; |
440 | template_str.push_str(name); | |
441 | } | |
442 | ||
443 | InlineAsmOperandRef::SymStatic { def_id } => { | |
04454e1e FG |
444 | // TODO(@Amanieu): Additional mangling is needed on |
445 | // some targets to add a leading underscore (Mach-O). | |
c295e0f8 XL |
446 | let instance = Instance::mono(self.tcx, def_id); |
447 | let name = self.tcx.symbol_name(instance).name; | |
448 | template_str.push_str(name); | |
449 | } | |
450 | ||
451 | InlineAsmOperandRef::Const { ref string } => { | |
452 | // Const operands get injected directly into the template | |
453 | if att_dialect { | |
454 | template_str.push('$'); | |
455 | } | |
456 | template_str.push_str(string); | |
457 | } | |
458 | } | |
459 | } | |
460 | } | |
461 | } | |
462 | ||
923072b8 | 463 | if att_dialect { |
c295e0f8 XL |
464 | template_str.push_str(INTEL_SYNTAX_INS); |
465 | } | |
a2a8927a | 466 | |
c295e0f8 XL |
467 | // 4. Generate Extended Asm block |
468 | ||
469 | let block = self.llbb(); | |
470 | let extended_asm = block.add_extended_asm(None, &template_str); | |
471 | ||
472 | for op in &outputs { | |
473 | extended_asm.add_output_operand(None, &op.to_constraint(), op.tmp_var); | |
474 | } | |
475 | ||
476 | for op in &inputs { | |
477 | extended_asm.add_input_operand(None, &op.constraint, op.val); | |
478 | } | |
479 | ||
480 | for clobber in clobbers.iter() { | |
481 | extended_asm.add_clobber(clobber); | |
482 | } | |
483 | ||
484 | if !options.contains(InlineAsmOptions::PRESERVES_FLAGS) { | |
a2a8927a | 485 | // TODO(@Commeownist): I'm not 100% sure this one clobber is sufficient |
c295e0f8 XL |
486 | // on all architectures. For instance, what about FP stack? |
487 | extended_asm.add_clobber("cc"); | |
488 | } | |
489 | if !options.contains(InlineAsmOptions::NOMEM) { | |
490 | extended_asm.add_clobber("memory"); | |
491 | } | |
492 | if !options.contains(InlineAsmOptions::PURE) { | |
493 | extended_asm.set_volatile_flag(true); | |
494 | } | |
495 | if !options.contains(InlineAsmOptions::NOSTACK) { | |
496 | // TODO(@Commeownist): figure out how to align stack | |
497 | } | |
498 | if options.contains(InlineAsmOptions::NORETURN) { | |
499 | let builtin_unreachable = self.context.get_builtin_function("__builtin_unreachable"); | |
500 | let builtin_unreachable: RValue<'gcc> = unsafe { std::mem::transmute(builtin_unreachable) }; | |
2b03887a | 501 | self.call(self.type_void(), None, builtin_unreachable, &[], None); |
c295e0f8 XL |
502 | } |
503 | ||
a2a8927a | 504 | // Write results to outputs. |
c295e0f8 XL |
505 | // |
506 | // We need to do this because: | |
a2a8927a | 507 | // 1. Turning `PlaceRef` into `RValue` is error-prone and has nasty edge cases |
c295e0f8 XL |
508 | // (especially with current `rustc_backend_ssa` API). |
509 | // 2. Not every output operand has an `out_place`, and it's required by `add_output_operand`. | |
510 | // | |
511 | // Instead, we generate a temporary output variable for each output operand, and then this loop, | |
512 | // generates `out_place = tmp_var;` assignments if out_place exists. | |
513 | for op in &outputs { | |
514 | if let Some(place) = op.out_place { | |
a2a8927a | 515 | OperandValue::Immediate(op.tmp_var.to_rvalue()).store(self, place); |
c295e0f8 XL |
516 | } |
517 | } | |
518 | ||
519 | } | |
520 | } | |
521 | ||
522 | fn estimate_template_length(template: &[InlineAsmTemplatePiece], constants_len: usize, att_dialect: bool) -> usize { | |
523 | let len: usize = template.iter().map(|piece| { | |
524 | match *piece { | |
525 | InlineAsmTemplatePiece::String(ref string) => { | |
526 | string.len() | |
527 | } | |
528 | InlineAsmTemplatePiece::Placeholder { .. } => { | |
529 | // '%' + 1 char modifier + 1 char index | |
530 | 3 | |
531 | } | |
532 | } | |
533 | }) | |
534 | .sum(); | |
535 | ||
536 | // increase it by 5% to account for possible '%' signs that'll be duplicated | |
537 | // I pulled the number out of blue, but should be fair enough | |
538 | // as the upper bound | |
539 | let mut res = (len as f32 * 1.05) as usize + constants_len; | |
540 | ||
541 | if att_dialect { | |
542 | res += INTEL_SYNTAX_INS.len() + ATT_SYNTAX_INS.len(); | |
543 | } | |
544 | res | |
545 | } | |
546 | ||
547 | /// Converts a register class to a GCC constraint code. | |
548 | fn reg_to_gcc(reg: InlineAsmRegOrRegClass) -> ConstraintOrRegister { | |
549 | let constraint = match reg { | |
550 | // For vector registers LLVM wants the register name to match the type size. | |
551 | InlineAsmRegOrRegClass::Reg(reg) => { | |
552 | match reg { | |
553 | InlineAsmReg::X86(_) => { | |
554 | // TODO(antoyo): add support for vector register. | |
555 | // | |
556 | // // For explicit registers, we have to create a register variable: https://stackoverflow.com/a/31774784/389119 | |
557 | return ConstraintOrRegister::Register(match reg.name() { | |
558 | // Some of registers' names does not map 1-1 from rust to gcc | |
559 | "st(0)" => "st", | |
560 | ||
561 | name => name, | |
562 | }); | |
563 | } | |
564 | ||
565 | _ => unimplemented!(), | |
566 | } | |
567 | }, | |
568 | InlineAsmRegOrRegClass::RegClass(reg) => match reg { | |
569 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => unimplemented!(), | |
570 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => unimplemented!(), | |
571 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) => unimplemented!(), | |
572 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => unimplemented!(), | |
573 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) => unimplemented!(), | |
c295e0f8 XL |
574 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg) |
575 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16) | |
576 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8) => unimplemented!(), | |
577 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16) | |
578 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8) | |
579 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4) => unimplemented!(), | |
580 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg) | |
581 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg) => unimplemented!(), | |
a2a8927a | 582 | InlineAsmRegClass::Avr(_) => unimplemented!(), |
c295e0f8 XL |
583 | InlineAsmRegClass::Bpf(_) => unimplemented!(), |
584 | InlineAsmRegClass::Hexagon(HexagonInlineAsmRegClass::reg) => unimplemented!(), | |
585 | InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg) => unimplemented!(), | |
586 | InlineAsmRegClass::Mips(MipsInlineAsmRegClass::freg) => unimplemented!(), | |
5099ac24 | 587 | InlineAsmRegClass::Msp430(_) => unimplemented!(), |
c295e0f8 XL |
588 | InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg16) => unimplemented!(), |
589 | InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg32) => unimplemented!(), | |
590 | InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg64) => unimplemented!(), | |
591 | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg) => unimplemented!(), | |
592 | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => unimplemented!(), | |
593 | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::freg) => unimplemented!(), | |
594 | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::cr) | |
595 | | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::xer) => { | |
596 | unreachable!("clobber-only") | |
597 | }, | |
598 | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) => unimplemented!(), | |
599 | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => unimplemented!(), | |
600 | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::vreg) => unimplemented!(), | |
601 | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg) => "r", | |
602 | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => "Q", | |
603 | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_byte) => "q", | |
604 | InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg) | |
605 | | InlineAsmRegClass::X86(X86InlineAsmRegClass::ymm_reg) => "x", | |
606 | InlineAsmRegClass::X86(X86InlineAsmRegClass::zmm_reg) => "v", | |
923072b8 | 607 | InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => "Yk", |
04454e1e | 608 | InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg0) => unimplemented!(), |
c295e0f8 XL |
609 | InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => unimplemented!(), |
610 | InlineAsmRegClass::X86( | |
923072b8 | 611 | X86InlineAsmRegClass::x87_reg | X86InlineAsmRegClass::mmx_reg | X86InlineAsmRegClass::tmm_reg, |
c295e0f8 XL |
612 | ) => unreachable!("clobber-only"), |
613 | InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { | |
614 | bug!("GCC backend does not support SPIR-V") | |
615 | } | |
616 | InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg) => unimplemented!(), | |
617 | InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => unimplemented!(), | |
618 | InlineAsmRegClass::Err => unreachable!(), | |
619 | } | |
620 | }; | |
621 | ||
622 | ConstraintOrRegister::Constraint(constraint) | |
623 | } | |
624 | ||
625 | /// Type to use for outputs that are discarded. It doesn't really matter what | |
626 | /// the type is, as long as it is valid for the constraint code. | |
627 | fn dummy_output_type<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, reg: InlineAsmRegClass) -> Type<'gcc> { | |
628 | match reg { | |
629 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => cx.type_i32(), | |
630 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => unimplemented!(), | |
631 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) | |
632 | | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => { | |
633 | unimplemented!() | |
634 | } | |
a2a8927a | 635 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg)=> cx.type_i32(), |
c295e0f8 XL |
636 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg) |
637 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16) => cx.type_f32(), | |
638 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg) | |
639 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16) | |
640 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8) => cx.type_f64(), | |
641 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg) | |
642 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8) | |
643 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4) => { | |
644 | unimplemented!() | |
645 | } | |
a2a8927a | 646 | InlineAsmRegClass::Avr(_) => unimplemented!(), |
c295e0f8 XL |
647 | InlineAsmRegClass::Bpf(_) => unimplemented!(), |
648 | InlineAsmRegClass::Hexagon(HexagonInlineAsmRegClass::reg) => cx.type_i32(), | |
649 | InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg) => cx.type_i32(), | |
650 | InlineAsmRegClass::Mips(MipsInlineAsmRegClass::freg) => cx.type_f32(), | |
5099ac24 | 651 | InlineAsmRegClass::Msp430(_) => unimplemented!(), |
c295e0f8 XL |
652 | InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg16) => cx.type_i16(), |
653 | InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg32) => cx.type_i32(), | |
654 | InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg64) => cx.type_i64(), | |
655 | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg) => cx.type_i32(), | |
656 | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => cx.type_i32(), | |
657 | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::freg) => cx.type_f64(), | |
658 | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::cr) | |
659 | | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::xer) => { | |
660 | unreachable!("clobber-only") | |
661 | }, | |
662 | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) => cx.type_i32(), | |
663 | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => cx.type_f32(), | |
664 | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::vreg) => cx.type_f32(), | |
665 | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg) | |
666 | | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => cx.type_i32(), | |
667 | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_byte) => cx.type_i8(), | |
668 | InlineAsmRegClass::X86(X86InlineAsmRegClass::mmx_reg) => unimplemented!(), | |
669 | InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg) | |
670 | | InlineAsmRegClass::X86(X86InlineAsmRegClass::ymm_reg) | |
671 | | InlineAsmRegClass::X86(X86InlineAsmRegClass::zmm_reg) => cx.type_f32(), | |
672 | InlineAsmRegClass::X86(X86InlineAsmRegClass::x87_reg) => unimplemented!(), | |
673 | InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => cx.type_i16(), | |
04454e1e | 674 | InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg0) => cx.type_i16(), |
923072b8 | 675 | InlineAsmRegClass::X86(X86InlineAsmRegClass::tmm_reg) => unimplemented!(), |
c295e0f8 XL |
676 | InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => cx.type_i32(), |
677 | InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { | |
678 | bug!("LLVM backend does not support SPIR-V") | |
679 | }, | |
680 | InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg) => cx.type_i32(), | |
681 | InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => cx.type_f64(), | |
682 | InlineAsmRegClass::Err => unreachable!(), | |
683 | } | |
684 | } | |
685 | ||
04454e1e FG |
686 | impl<'gcc, 'tcx> AsmMethods<'tcx> for CodegenCx<'gcc, 'tcx> { |
687 | fn codegen_global_asm(&self, template: &[InlineAsmTemplatePiece], operands: &[GlobalAsmOperandRef<'tcx>], options: InlineAsmOptions, _line_spans: &[Span]) { | |
c295e0f8 XL |
688 | let asm_arch = self.tcx.sess.asm_arch.unwrap(); |
689 | ||
690 | // Default to Intel syntax on x86 | |
923072b8 FG |
691 | let att_dialect = matches!(asm_arch, InlineAsmArch::X86 | InlineAsmArch::X86_64) |
692 | && options.contains(InlineAsmOptions::ATT_SYNTAX); | |
c295e0f8 XL |
693 | |
694 | // Build the template string | |
695 | let mut template_str = String::new(); | |
696 | for piece in template { | |
697 | match *piece { | |
698 | InlineAsmTemplatePiece::String(ref string) => { | |
699 | for line in string.lines() { | |
700 | // NOTE: gcc does not allow inline comment, so remove them. | |
701 | let line = | |
702 | if let Some(index) = line.rfind("//") { | |
703 | &line[..index] | |
704 | } | |
705 | else { | |
706 | line | |
707 | }; | |
708 | template_str.push_str(line); | |
709 | template_str.push('\n'); | |
710 | } | |
711 | }, | |
712 | InlineAsmTemplatePiece::Placeholder { operand_idx, modifier: _, span: _ } => { | |
713 | match operands[operand_idx] { | |
714 | GlobalAsmOperandRef::Const { ref string } => { | |
715 | // Const operands get injected directly into the | |
716 | // template. Note that we don't need to escape % | |
717 | // here unlike normal inline assembly. | |
718 | template_str.push_str(string); | |
719 | } | |
04454e1e FG |
720 | |
721 | GlobalAsmOperandRef::SymFn { instance } => { | |
722 | // TODO(@Amanieu): Additional mangling is needed on | |
723 | // some targets to add a leading underscore (Mach-O) | |
724 | // or byte count suffixes (x86 Windows). | |
725 | let name = self.tcx.symbol_name(instance).name; | |
726 | template_str.push_str(name); | |
727 | } | |
728 | ||
729 | GlobalAsmOperandRef::SymStatic { def_id } => { | |
730 | // TODO(@Amanieu): Additional mangling is needed on | |
731 | // some targets to add a leading underscore (Mach-O). | |
732 | let instance = Instance::mono(self.tcx, def_id); | |
733 | let name = self.tcx.symbol_name(instance).name; | |
734 | template_str.push_str(name); | |
735 | } | |
c295e0f8 XL |
736 | } |
737 | } | |
738 | } | |
739 | } | |
740 | ||
741 | let template_str = | |
923072b8 FG |
742 | if att_dialect { |
743 | format!(".att_syntax\n\t{}\n\t.intel_syntax noprefix", template_str) | |
c295e0f8 XL |
744 | } |
745 | else { | |
923072b8 | 746 | template_str |
c295e0f8 XL |
747 | }; |
748 | // NOTE: seems like gcc will put the asm in the wrong section, so set it to .text manually. | |
749 | let template_str = format!(".pushsection .text\n{}\n.popsection", template_str); | |
750 | self.context.add_top_level_asm(None, &template_str); | |
751 | } | |
752 | } | |
753 | ||
754 | fn modifier_to_gcc(arch: InlineAsmArch, reg: InlineAsmRegClass, modifier: Option<char>) -> Option<char> { | |
755 | match reg { | |
756 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => modifier, | |
757 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => modifier, | |
758 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) | |
759 | | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => { | |
760 | unimplemented!() | |
761 | } | |
a2a8927a | 762 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) => unimplemented!(), |
c295e0f8 XL |
763 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg) |
764 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16) => unimplemented!(), | |
765 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg) | |
766 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16) | |
767 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8) => unimplemented!(), | |
768 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg) | |
769 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8) | |
770 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4) => { | |
771 | unimplemented!() | |
772 | } | |
a2a8927a | 773 | InlineAsmRegClass::Avr(_) => unimplemented!(), |
c295e0f8 XL |
774 | InlineAsmRegClass::Bpf(_) => unimplemented!(), |
775 | InlineAsmRegClass::Hexagon(_) => unimplemented!(), | |
776 | InlineAsmRegClass::Mips(_) => unimplemented!(), | |
5099ac24 | 777 | InlineAsmRegClass::Msp430(_) => unimplemented!(), |
c295e0f8 XL |
778 | InlineAsmRegClass::Nvptx(_) => unimplemented!(), |
779 | InlineAsmRegClass::PowerPC(_) => unimplemented!(), | |
780 | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) | |
781 | | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => unimplemented!(), | |
782 | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::vreg) => unimplemented!(), | |
783 | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg) | |
784 | | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => match modifier { | |
785 | None => if arch == InlineAsmArch::X86_64 { Some('q') } else { Some('k') }, | |
786 | Some('l') => Some('b'), | |
787 | Some('h') => Some('h'), | |
788 | Some('x') => Some('w'), | |
789 | Some('e') => Some('k'), | |
790 | Some('r') => Some('q'), | |
791 | _ => unreachable!(), | |
792 | }, | |
793 | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_byte) => None, | |
794 | InlineAsmRegClass::X86(reg @ X86InlineAsmRegClass::xmm_reg) | |
795 | | InlineAsmRegClass::X86(reg @ X86InlineAsmRegClass::ymm_reg) | |
796 | | InlineAsmRegClass::X86(reg @ X86InlineAsmRegClass::zmm_reg) => match (reg, modifier) { | |
797 | (X86InlineAsmRegClass::xmm_reg, None) => Some('x'), | |
798 | (X86InlineAsmRegClass::ymm_reg, None) => Some('t'), | |
799 | (X86InlineAsmRegClass::zmm_reg, None) => Some('g'), | |
800 | (_, Some('x')) => Some('x'), | |
801 | (_, Some('y')) => Some('t'), | |
802 | (_, Some('z')) => Some('g'), | |
803 | _ => unreachable!(), | |
804 | }, | |
805 | InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => None, | |
04454e1e | 806 | InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg0) => None, |
923072b8 | 807 | InlineAsmRegClass::X86(X86InlineAsmRegClass::x87_reg | X86InlineAsmRegClass::mmx_reg | X86InlineAsmRegClass::tmm_reg) => { |
c295e0f8 XL |
808 | unreachable!("clobber-only") |
809 | } | |
810 | InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => unimplemented!(), | |
811 | InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { | |
812 | bug!("LLVM backend does not support SPIR-V") | |
813 | }, | |
814 | InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg) => unimplemented!(), | |
815 | InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => unimplemented!(), | |
816 | InlineAsmRegClass::Err => unreachable!(), | |
817 | } | |
818 | } |