]>
Commit | Line | Data |
---|---|---|
c295e0f8 XL |
1 | use gccjit::{LValue, RValue, ToRValue, Type}; |
2 | use rustc_ast::ast::{InlineAsmOptions, InlineAsmTemplatePiece}; | |
3 | use rustc_codegen_ssa::mir::operand::OperandValue; | |
4 | use rustc_codegen_ssa::mir::place::PlaceRef; | |
5 | use rustc_codegen_ssa::traits::{AsmBuilderMethods, AsmMethods, BaseTypeMethods, BuilderMethods, GlobalAsmOperandRef, InlineAsmOperandRef}; | |
6 | ||
c295e0f8 | 7 | use rustc_middle::{bug, ty::Instance}; |
5099ac24 | 8 | use rustc_span::Span; |
c295e0f8 XL |
9 | use rustc_target::asm::*; |
10 | ||
11 | use std::borrow::Cow; | |
12 | ||
13 | use crate::builder::Builder; | |
14 | use crate::context::CodegenCx; | |
15 | use crate::type_of::LayoutGccExt; | |
923072b8 | 16 | use crate::callee::get_fn; |
c295e0f8 XL |
17 | |
18 | ||
19 | // Rust asm! and GCC Extended Asm semantics differ substantially. | |
20 | // | |
a2a8927a XL |
21 | // 1. Rust asm operands go along as one list of operands. Operands themselves indicate |
22 | // if they're "in" or "out". "In" and "out" operands can interleave. One operand can be | |
c295e0f8 XL |
23 | // both "in" and "out" (`inout(reg)`). |
24 | // | |
a2a8927a XL |
25 | // GCC asm has two different lists for "in" and "out" operands. In terms of gccjit, |
26 | // this means that all "out" operands must go before "in" operands. "In" and "out" operands | |
c295e0f8 XL |
27 | // cannot interleave. |
28 | // | |
a2a8927a | 29 | // 2. Operand lists in both Rust and GCC are indexed. Index starts from 0. Indexes are important |
c295e0f8 XL |
30 | // because the asm template refers to operands by index. |
31 | // | |
32 | // Mapping from Rust to GCC index would be 1-1 if it wasn't for... | |
33 | // | |
a2a8927a XL |
34 | // 3. Clobbers. GCC has a separate list of clobbers, and clobbers don't have indexes. |
35 | // Contrary, Rust expresses clobbers through "out" operands that aren't tied to | |
c295e0f8 XL |
36 | // a variable (`_`), and such "clobbers" do have index. |
37 | // | |
a2a8927a XL |
38 | // 4. Furthermore, GCC Extended Asm does not support explicit register constraints |
39 | // (like `out("eax")`) directly, offering so-called "local register variables" | |
40 | // as a workaround. These variables need to be declared and initialized *before* | |
41 | // the Extended Asm block but *after* normal local variables | |
c295e0f8 XL |
42 | // (see comment in `codegen_inline_asm` for explanation). |
43 | // | |
a2a8927a | 44 | // With that in mind, let's see how we translate Rust syntax to GCC |
c295e0f8 XL |
45 | // (from now on, `CC` stands for "constraint code"): |
46 | // | |
47 | // * `out(reg_class) var` -> translated to output operand: `"=CC"(var)` | |
48 | // * `inout(reg_class) var` -> translated to output operand: `"+CC"(var)` | |
49 | // * `in(reg_class) var` -> translated to input operand: `"CC"(var)` | |
50 | // | |
51 | // * `out(reg_class) _` -> translated to one `=r(tmp)`, where "tmp" is a temporary unused variable | |
52 | // | |
53 | // * `out("explicit register") _` -> not translated to any operands, register is simply added to clobbers list | |
54 | // | |
a2a8927a | 55 | // * `inout(reg_class) in_var => out_var` -> translated to two operands: |
c295e0f8 | 56 | // output: `"=CC"(in_var)` |
a2a8927a | 57 | // input: `"num"(out_var)` where num is the GCC index |
c295e0f8 XL |
58 | // of the corresponding output operand |
59 | // | |
a2a8927a | 60 | // * `inout(reg_class) in_var => _` -> same as `inout(reg_class) in_var => tmp`, |
c295e0f8 XL |
61 | // where "tmp" is a temporary unused variable |
62 | // | |
a2a8927a XL |
63 | // * `out/in/inout("explicit register") var` -> translated to one or two operands as described above |
64 | // with `"r"(var)` constraint, | |
c295e0f8 | 65 | // and one register variable assigned to the desired register. |
c295e0f8 XL |
66 | |
67 | const ATT_SYNTAX_INS: &str = ".att_syntax noprefix\n\t"; | |
68 | const INTEL_SYNTAX_INS: &str = "\n\t.intel_syntax noprefix"; | |
69 | ||
70 | ||
71 | struct AsmOutOperand<'a, 'tcx, 'gcc> { | |
72 | rust_idx: usize, | |
73 | constraint: &'a str, | |
74 | late: bool, | |
75 | readwrite: bool, | |
76 | ||
77 | tmp_var: LValue<'gcc>, | |
78 | out_place: Option<PlaceRef<'tcx, RValue<'gcc>>> | |
79 | } | |
80 | ||
81 | struct AsmInOperand<'a, 'tcx> { | |
82 | rust_idx: usize, | |
83 | constraint: Cow<'a, str>, | |
84 | val: RValue<'tcx> | |
85 | } | |
86 | ||
87 | impl AsmOutOperand<'_, '_, '_> { | |
88 | fn to_constraint(&self) -> String { | |
89 | let mut res = String::with_capacity(self.constraint.len() + self.late as usize + 1); | |
90 | ||
91 | let sign = if self.readwrite { '+' } else { '=' }; | |
92 | res.push(sign); | |
93 | if !self.late { | |
94 | res.push('&'); | |
95 | } | |
96 | ||
97 | res.push_str(&self.constraint); | |
98 | res | |
99 | } | |
100 | } | |
101 | ||
102 | enum ConstraintOrRegister { | |
103 | Constraint(&'static str), | |
104 | Register(&'static str) | |
105 | } | |
106 | ||
107 | ||
108 | impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> { | |
a2a8927a XL |
109 | fn codegen_inline_asm(&mut self, template: &[InlineAsmTemplatePiece], rust_operands: &[InlineAsmOperandRef<'tcx, Self>], options: InlineAsmOptions, span: &[Span], _instance: Instance<'_>, _dest_catch_funclet: Option<(Self::BasicBlock, Self::BasicBlock, Option<&Self::Funclet>)>) { |
110 | if options.contains(InlineAsmOptions::MAY_UNWIND) { | |
111 | self.sess() | |
112 | .struct_span_err(span[0], "GCC backend does not support unwinding from inline asm") | |
113 | .emit(); | |
114 | return; | |
115 | } | |
116 | ||
c295e0f8 XL |
117 | let asm_arch = self.tcx.sess.asm_arch.unwrap(); |
118 | let is_x86 = matches!(asm_arch, InlineAsmArch::X86 | InlineAsmArch::X86_64); | |
119 | let att_dialect = is_x86 && options.contains(InlineAsmOptions::ATT_SYNTAX); | |
c295e0f8 | 120 | |
a2a8927a | 121 | // GCC index of an output operand equals its position in the array |
c295e0f8 XL |
122 | let mut outputs = vec![]; |
123 | ||
124 | // GCC index of an input operand equals its position in the array | |
125 | // added to `outputs.len()` | |
126 | let mut inputs = vec![]; | |
127 | ||
128 | // Clobbers collected from `out("explicit register") _` and `inout("expl_reg") var => _` | |
129 | let mut clobbers = vec![]; | |
130 | ||
131 | // We're trying to preallocate space for the template | |
132 | let mut constants_len = 0; | |
133 | ||
134 | // There are rules we must adhere to if we want GCC to do the right thing: | |
a2a8927a | 135 | // |
c295e0f8 | 136 | // * Every local variable that the asm block uses as an output must be declared *before* |
a2a8927a | 137 | // the asm block. |
c295e0f8 XL |
138 | // * There must be no instructions whatsoever between the register variables and the asm. |
139 | // | |
140 | // Therefore, the backend must generate the instructions strictly in this order: | |
141 | // | |
142 | // 1. Output variables. | |
143 | // 2. Register variables. | |
144 | // 3. The asm block. | |
145 | // | |
146 | // We also must make sure that no input operands are emitted before output operands. | |
147 | // | |
148 | // This is why we work in passes, first emitting local vars, then local register vars. | |
a2a8927a | 149 | // Also, we don't emit any asm operands immediately; we save them to |
c295e0f8 XL |
150 | // the one of the buffers to be emitted later. |
151 | ||
152 | // 1. Normal variables (and saving operands to buffers). | |
153 | for (rust_idx, op) in rust_operands.iter().enumerate() { | |
154 | match *op { | |
155 | InlineAsmOperandRef::Out { reg, late, place } => { | |
156 | use ConstraintOrRegister::*; | |
157 | ||
158 | let (constraint, ty) = match (reg_to_gcc(reg), place) { | |
159 | (Constraint(constraint), Some(place)) => (constraint, place.layout.gcc_type(self.cx, false)), | |
160 | // When `reg` is a class and not an explicit register but the out place is not specified, | |
161 | // we need to create an unused output variable to assign the output to. This var | |
a2a8927a | 162 | // needs to be of a type that's "compatible" with the register class, but specific type |
c295e0f8 XL |
163 | // doesn't matter. |
164 | (Constraint(constraint), None) => (constraint, dummy_output_type(self.cx, reg.reg_class())), | |
165 | (Register(_), Some(_)) => { | |
166 | // left for the next pass | |
167 | continue | |
168 | }, | |
169 | (Register(reg_name), None) => { | |
170 | // `clobber_abi` can add lots of clobbers that are not supported by the target, | |
171 | // such as AVX-512 registers, so we just ignore unsupported registers | |
172 | let is_target_supported = reg.reg_class().supported_types(asm_arch).iter() | |
173 | .any(|&(_, feature)| { | |
174 | if let Some(feature) = feature { | |
5099ac24 | 175 | self.tcx.sess.target_features.contains(&feature) |
c295e0f8 XL |
176 | } else { |
177 | true // Register class is unconditionally supported | |
178 | } | |
179 | }); | |
180 | ||
181 | if is_target_supported && !clobbers.contains(®_name) { | |
182 | clobbers.push(reg_name); | |
183 | } | |
184 | continue | |
185 | } | |
186 | }; | |
187 | ||
188 | let tmp_var = self.current_func().new_local(None, ty, "output_register"); | |
189 | outputs.push(AsmOutOperand { | |
a2a8927a | 190 | constraint, |
c295e0f8 XL |
191 | rust_idx, |
192 | late, | |
193 | readwrite: false, | |
194 | tmp_var, | |
195 | out_place: place | |
196 | }); | |
197 | } | |
198 | ||
199 | InlineAsmOperandRef::In { reg, value } => { | |
200 | if let ConstraintOrRegister::Constraint(constraint) = reg_to_gcc(reg) { | |
a2a8927a XL |
201 | inputs.push(AsmInOperand { |
202 | constraint: Cow::Borrowed(constraint), | |
203 | rust_idx, | |
c295e0f8 XL |
204 | val: value.immediate() |
205 | }); | |
a2a8927a | 206 | } |
c295e0f8 XL |
207 | else { |
208 | // left for the next pass | |
209 | continue | |
210 | } | |
211 | } | |
212 | ||
213 | InlineAsmOperandRef::InOut { reg, late, in_value, out_place } => { | |
214 | let constraint = if let ConstraintOrRegister::Constraint(constraint) = reg_to_gcc(reg) { | |
215 | constraint | |
a2a8927a | 216 | } |
c295e0f8 XL |
217 | else { |
218 | // left for the next pass | |
219 | continue | |
220 | }; | |
221 | ||
222 | // Rustc frontend guarantees that input and output types are "compatible", | |
223 | // so we can just use input var's type for the output variable. | |
224 | // | |
a2a8927a XL |
225 | // This decision is also backed by the fact that LLVM needs in and out |
226 | // values to be of *exactly the same type*, not just "compatible". | |
c295e0f8 XL |
227 | // I'm not sure if GCC is so picky too, but better safe than sorry. |
228 | let ty = in_value.layout.gcc_type(self.cx, false); | |
229 | let tmp_var = self.current_func().new_local(None, ty, "output_register"); | |
230 | ||
231 | // If the out_place is None (i.e `inout(reg) _` syntax was used), we translate | |
a2a8927a | 232 | // it to one "readwrite (+) output variable", otherwise we translate it to two |
c295e0f8 XL |
233 | // "out and tied in" vars as described above. |
234 | let readwrite = out_place.is_none(); | |
235 | outputs.push(AsmOutOperand { | |
a2a8927a | 236 | constraint, |
c295e0f8 XL |
237 | rust_idx, |
238 | late, | |
239 | readwrite, | |
a2a8927a | 240 | tmp_var, |
c295e0f8 XL |
241 | out_place, |
242 | }); | |
243 | ||
244 | if !readwrite { | |
245 | let out_gcc_idx = outputs.len() - 1; | |
246 | let constraint = Cow::Owned(out_gcc_idx.to_string()); | |
247 | ||
248 | inputs.push(AsmInOperand { | |
a2a8927a XL |
249 | constraint, |
250 | rust_idx, | |
c295e0f8 XL |
251 | val: in_value.immediate() |
252 | }); | |
253 | } | |
254 | } | |
255 | ||
256 | InlineAsmOperandRef::Const { ref string } => { | |
257 | constants_len += string.len() + att_dialect as usize; | |
258 | } | |
259 | ||
260 | InlineAsmOperandRef::SymFn { instance } => { | |
04454e1e FG |
261 | // TODO(@Amanieu): Additional mangling is needed on |
262 | // some targets to add a leading underscore (Mach-O) | |
263 | // or byte count suffixes (x86 Windows). | |
c295e0f8 XL |
264 | constants_len += self.tcx.symbol_name(instance).name.len(); |
265 | } | |
266 | InlineAsmOperandRef::SymStatic { def_id } => { | |
04454e1e FG |
267 | // TODO(@Amanieu): Additional mangling is needed on |
268 | // some targets to add a leading underscore (Mach-O). | |
c295e0f8 XL |
269 | constants_len += self.tcx.symbol_name(Instance::mono(self.tcx, def_id)).name.len(); |
270 | } | |
271 | } | |
272 | } | |
273 | ||
274 | // 2. Register variables. | |
275 | for (rust_idx, op) in rust_operands.iter().enumerate() { | |
276 | match *op { | |
277 | // `out("explicit register") var` | |
278 | InlineAsmOperandRef::Out { reg, late, place } => { | |
279 | if let ConstraintOrRegister::Register(reg_name) = reg_to_gcc(reg) { | |
280 | let out_place = if let Some(place) = place { | |
281 | place | |
a2a8927a | 282 | } |
c295e0f8 XL |
283 | else { |
284 | // processed in the previous pass | |
285 | continue | |
286 | }; | |
287 | ||
288 | let ty = out_place.layout.gcc_type(self.cx, false); | |
289 | let tmp_var = self.current_func().new_local(None, ty, "output_register"); | |
290 | tmp_var.set_register_name(reg_name); | |
291 | ||
292 | outputs.push(AsmOutOperand { | |
a2a8927a | 293 | constraint: "r".into(), |
c295e0f8 XL |
294 | rust_idx, |
295 | late, | |
296 | readwrite: false, | |
297 | tmp_var, | |
298 | out_place: Some(out_place) | |
299 | }); | |
300 | } | |
301 | ||
302 | // processed in the previous pass | |
303 | } | |
304 | ||
305 | // `in("explicit register") var` | |
306 | InlineAsmOperandRef::In { reg, value } => { | |
307 | if let ConstraintOrRegister::Register(reg_name) = reg_to_gcc(reg) { | |
308 | let ty = value.layout.gcc_type(self.cx, false); | |
309 | let reg_var = self.current_func().new_local(None, ty, "input_register"); | |
310 | reg_var.set_register_name(reg_name); | |
311 | self.llbb().add_assignment(None, reg_var, value.immediate()); | |
312 | ||
a2a8927a XL |
313 | inputs.push(AsmInOperand { |
314 | constraint: "r".into(), | |
315 | rust_idx, | |
c295e0f8 XL |
316 | val: reg_var.to_rvalue() |
317 | }); | |
318 | } | |
319 | ||
320 | // processed in the previous pass | |
321 | } | |
322 | ||
323 | // `inout("explicit register") in_var => out_var` | |
324 | InlineAsmOperandRef::InOut { reg, late, in_value, out_place } => { | |
325 | if let ConstraintOrRegister::Register(reg_name) = reg_to_gcc(reg) { | |
c295e0f8 XL |
326 | // See explanation in the first pass. |
327 | let ty = in_value.layout.gcc_type(self.cx, false); | |
328 | let tmp_var = self.current_func().new_local(None, ty, "output_register"); | |
329 | tmp_var.set_register_name(reg_name); | |
330 | ||
331 | outputs.push(AsmOutOperand { | |
a2a8927a | 332 | constraint: "r".into(), |
c295e0f8 XL |
333 | rust_idx, |
334 | late, | |
335 | readwrite: false, | |
336 | tmp_var, | |
a2a8927a | 337 | out_place, |
c295e0f8 XL |
338 | }); |
339 | ||
340 | let constraint = Cow::Owned((outputs.len() - 1).to_string()); | |
a2a8927a XL |
341 | inputs.push(AsmInOperand { |
342 | constraint, | |
c295e0f8 XL |
343 | rust_idx, |
344 | val: in_value.immediate() | |
345 | }); | |
346 | } | |
347 | ||
348 | // processed in the previous pass | |
349 | } | |
350 | ||
923072b8 FG |
351 | InlineAsmOperandRef::SymFn { instance } => { |
352 | inputs.push(AsmInOperand { | |
353 | constraint: "X".into(), | |
354 | rust_idx, | |
355 | val: self.cx.rvalue_as_function(get_fn(self.cx, instance)) | |
356 | .get_address(None), | |
357 | }); | |
358 | } | |
359 | ||
360 | InlineAsmOperandRef::SymStatic { def_id } => { | |
361 | inputs.push(AsmInOperand { | |
362 | constraint: "X".into(), | |
363 | rust_idx, | |
364 | val: self.cx.get_static(def_id).get_address(None), | |
365 | }); | |
366 | } | |
367 | ||
368 | InlineAsmOperandRef::Const { .. } => { | |
c295e0f8 XL |
369 | // processed in the previous pass |
370 | } | |
371 | } | |
372 | } | |
373 | ||
374 | // 3. Build the template string | |
375 | ||
376 | let mut template_str = String::with_capacity(estimate_template_length(template, constants_len, att_dialect)); | |
923072b8 | 377 | if att_dialect { |
c295e0f8 XL |
378 | template_str.push_str(ATT_SYNTAX_INS); |
379 | } | |
380 | ||
381 | for piece in template { | |
382 | match *piece { | |
383 | InlineAsmTemplatePiece::String(ref string) => { | |
384 | // TODO(@Commeownist): switch to `Iterator::intersperse` once it's stable | |
385 | let mut iter = string.split('%'); | |
386 | if let Some(s) = iter.next() { | |
387 | template_str.push_str(s); | |
388 | } | |
389 | ||
390 | for s in iter { | |
391 | template_str.push_str("%%"); | |
392 | template_str.push_str(s); | |
393 | } | |
394 | } | |
395 | InlineAsmTemplatePiece::Placeholder { operand_idx, modifier, span: _ } => { | |
396 | let mut push_to_template = |modifier, gcc_idx| { | |
397 | use std::fmt::Write; | |
398 | ||
399 | template_str.push('%'); | |
400 | if let Some(modifier) = modifier { | |
401 | template_str.push(modifier); | |
402 | } | |
403 | write!(template_str, "{}", gcc_idx).expect("pushing to string failed"); | |
404 | }; | |
405 | ||
406 | match rust_operands[operand_idx] { | |
407 | InlineAsmOperandRef::Out { reg, .. } => { | |
408 | let modifier = modifier_to_gcc(asm_arch, reg.reg_class(), modifier); | |
409 | let gcc_index = outputs.iter() | |
410 | .position(|op| operand_idx == op.rust_idx) | |
411 | .expect("wrong rust index"); | |
412 | push_to_template(modifier, gcc_index); | |
413 | } | |
414 | ||
415 | InlineAsmOperandRef::In { reg, .. } => { | |
416 | let modifier = modifier_to_gcc(asm_arch, reg.reg_class(), modifier); | |
417 | let in_gcc_index = inputs.iter() | |
418 | .position(|op| operand_idx == op.rust_idx) | |
419 | .expect("wrong rust index"); | |
420 | let gcc_index = in_gcc_index + outputs.len(); | |
421 | push_to_template(modifier, gcc_index); | |
422 | } | |
423 | ||
424 | InlineAsmOperandRef::InOut { reg, .. } => { | |
425 | let modifier = modifier_to_gcc(asm_arch, reg.reg_class(), modifier); | |
426 | ||
427 | // The input register is tied to the output, so we can just use the index of the output register | |
428 | let gcc_index = outputs.iter() | |
429 | .position(|op| operand_idx == op.rust_idx) | |
430 | .expect("wrong rust index"); | |
431 | push_to_template(modifier, gcc_index); | |
432 | } | |
433 | ||
434 | InlineAsmOperandRef::SymFn { instance } => { | |
04454e1e FG |
435 | // TODO(@Amanieu): Additional mangling is needed on |
436 | // some targets to add a leading underscore (Mach-O) | |
437 | // or byte count suffixes (x86 Windows). | |
c295e0f8 XL |
438 | let name = self.tcx.symbol_name(instance).name; |
439 | template_str.push_str(name); | |
440 | } | |
441 | ||
442 | InlineAsmOperandRef::SymStatic { def_id } => { | |
04454e1e FG |
443 | // TODO(@Amanieu): Additional mangling is needed on |
444 | // some targets to add a leading underscore (Mach-O). | |
c295e0f8 XL |
445 | let instance = Instance::mono(self.tcx, def_id); |
446 | let name = self.tcx.symbol_name(instance).name; | |
447 | template_str.push_str(name); | |
448 | } | |
449 | ||
450 | InlineAsmOperandRef::Const { ref string } => { | |
451 | // Const operands get injected directly into the template | |
452 | if att_dialect { | |
453 | template_str.push('$'); | |
454 | } | |
455 | template_str.push_str(string); | |
456 | } | |
457 | } | |
458 | } | |
459 | } | |
460 | } | |
461 | ||
923072b8 | 462 | if att_dialect { |
c295e0f8 XL |
463 | template_str.push_str(INTEL_SYNTAX_INS); |
464 | } | |
a2a8927a | 465 | |
c295e0f8 XL |
466 | // 4. Generate Extended Asm block |
467 | ||
468 | let block = self.llbb(); | |
469 | let extended_asm = block.add_extended_asm(None, &template_str); | |
470 | ||
471 | for op in &outputs { | |
472 | extended_asm.add_output_operand(None, &op.to_constraint(), op.tmp_var); | |
473 | } | |
474 | ||
475 | for op in &inputs { | |
476 | extended_asm.add_input_operand(None, &op.constraint, op.val); | |
477 | } | |
478 | ||
479 | for clobber in clobbers.iter() { | |
480 | extended_asm.add_clobber(clobber); | |
481 | } | |
482 | ||
483 | if !options.contains(InlineAsmOptions::PRESERVES_FLAGS) { | |
a2a8927a | 484 | // TODO(@Commeownist): I'm not 100% sure this one clobber is sufficient |
c295e0f8 XL |
485 | // on all architectures. For instance, what about FP stack? |
486 | extended_asm.add_clobber("cc"); | |
487 | } | |
488 | if !options.contains(InlineAsmOptions::NOMEM) { | |
489 | extended_asm.add_clobber("memory"); | |
490 | } | |
491 | if !options.contains(InlineAsmOptions::PURE) { | |
492 | extended_asm.set_volatile_flag(true); | |
493 | } | |
494 | if !options.contains(InlineAsmOptions::NOSTACK) { | |
495 | // TODO(@Commeownist): figure out how to align stack | |
496 | } | |
497 | if options.contains(InlineAsmOptions::NORETURN) { | |
498 | let builtin_unreachable = self.context.get_builtin_function("__builtin_unreachable"); | |
499 | let builtin_unreachable: RValue<'gcc> = unsafe { std::mem::transmute(builtin_unreachable) }; | |
500 | self.call(self.type_void(), builtin_unreachable, &[], None); | |
501 | } | |
502 | ||
a2a8927a | 503 | // Write results to outputs. |
c295e0f8 XL |
504 | // |
505 | // We need to do this because: | |
a2a8927a | 506 | // 1. Turning `PlaceRef` into `RValue` is error-prone and has nasty edge cases |
c295e0f8 XL |
507 | // (especially with current `rustc_backend_ssa` API). |
508 | // 2. Not every output operand has an `out_place`, and it's required by `add_output_operand`. | |
509 | // | |
510 | // Instead, we generate a temporary output variable for each output operand, and then this loop, | |
511 | // generates `out_place = tmp_var;` assignments if out_place exists. | |
512 | for op in &outputs { | |
513 | if let Some(place) = op.out_place { | |
a2a8927a | 514 | OperandValue::Immediate(op.tmp_var.to_rvalue()).store(self, place); |
c295e0f8 XL |
515 | } |
516 | } | |
517 | ||
518 | } | |
519 | } | |
520 | ||
521 | fn estimate_template_length(template: &[InlineAsmTemplatePiece], constants_len: usize, att_dialect: bool) -> usize { | |
522 | let len: usize = template.iter().map(|piece| { | |
523 | match *piece { | |
524 | InlineAsmTemplatePiece::String(ref string) => { | |
525 | string.len() | |
526 | } | |
527 | InlineAsmTemplatePiece::Placeholder { .. } => { | |
528 | // '%' + 1 char modifier + 1 char index | |
529 | 3 | |
530 | } | |
531 | } | |
532 | }) | |
533 | .sum(); | |
534 | ||
535 | // increase it by 5% to account for possible '%' signs that'll be duplicated | |
536 | // I pulled the number out of blue, but should be fair enough | |
537 | // as the upper bound | |
538 | let mut res = (len as f32 * 1.05) as usize + constants_len; | |
539 | ||
540 | if att_dialect { | |
541 | res += INTEL_SYNTAX_INS.len() + ATT_SYNTAX_INS.len(); | |
542 | } | |
543 | res | |
544 | } | |
545 | ||
546 | /// Converts a register class to a GCC constraint code. | |
547 | fn reg_to_gcc(reg: InlineAsmRegOrRegClass) -> ConstraintOrRegister { | |
548 | let constraint = match reg { | |
549 | // For vector registers LLVM wants the register name to match the type size. | |
550 | InlineAsmRegOrRegClass::Reg(reg) => { | |
551 | match reg { | |
552 | InlineAsmReg::X86(_) => { | |
553 | // TODO(antoyo): add support for vector register. | |
554 | // | |
555 | // // For explicit registers, we have to create a register variable: https://stackoverflow.com/a/31774784/389119 | |
556 | return ConstraintOrRegister::Register(match reg.name() { | |
557 | // Some of registers' names does not map 1-1 from rust to gcc | |
558 | "st(0)" => "st", | |
559 | ||
560 | name => name, | |
561 | }); | |
562 | } | |
563 | ||
564 | _ => unimplemented!(), | |
565 | } | |
566 | }, | |
567 | InlineAsmRegOrRegClass::RegClass(reg) => match reg { | |
568 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => unimplemented!(), | |
569 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => unimplemented!(), | |
570 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) => unimplemented!(), | |
571 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => unimplemented!(), | |
572 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) => unimplemented!(), | |
c295e0f8 XL |
573 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg) |
574 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16) | |
575 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8) => unimplemented!(), | |
576 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16) | |
577 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8) | |
578 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4) => unimplemented!(), | |
579 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg) | |
580 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg) => unimplemented!(), | |
a2a8927a | 581 | InlineAsmRegClass::Avr(_) => unimplemented!(), |
c295e0f8 XL |
582 | InlineAsmRegClass::Bpf(_) => unimplemented!(), |
583 | InlineAsmRegClass::Hexagon(HexagonInlineAsmRegClass::reg) => unimplemented!(), | |
584 | InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg) => unimplemented!(), | |
585 | InlineAsmRegClass::Mips(MipsInlineAsmRegClass::freg) => unimplemented!(), | |
5099ac24 | 586 | InlineAsmRegClass::Msp430(_) => unimplemented!(), |
c295e0f8 XL |
587 | InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg16) => unimplemented!(), |
588 | InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg32) => unimplemented!(), | |
589 | InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg64) => unimplemented!(), | |
590 | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg) => unimplemented!(), | |
591 | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => unimplemented!(), | |
592 | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::freg) => unimplemented!(), | |
593 | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::cr) | |
594 | | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::xer) => { | |
595 | unreachable!("clobber-only") | |
596 | }, | |
597 | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) => unimplemented!(), | |
598 | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => unimplemented!(), | |
599 | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::vreg) => unimplemented!(), | |
600 | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg) => "r", | |
601 | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => "Q", | |
602 | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_byte) => "q", | |
603 | InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg) | |
604 | | InlineAsmRegClass::X86(X86InlineAsmRegClass::ymm_reg) => "x", | |
605 | InlineAsmRegClass::X86(X86InlineAsmRegClass::zmm_reg) => "v", | |
923072b8 | 606 | InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => "Yk", |
04454e1e | 607 | InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg0) => unimplemented!(), |
c295e0f8 XL |
608 | InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => unimplemented!(), |
609 | InlineAsmRegClass::X86( | |
923072b8 | 610 | X86InlineAsmRegClass::x87_reg | X86InlineAsmRegClass::mmx_reg | X86InlineAsmRegClass::tmm_reg, |
c295e0f8 XL |
611 | ) => unreachable!("clobber-only"), |
612 | InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { | |
613 | bug!("GCC backend does not support SPIR-V") | |
614 | } | |
615 | InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg) => unimplemented!(), | |
616 | InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => unimplemented!(), | |
617 | InlineAsmRegClass::Err => unreachable!(), | |
618 | } | |
619 | }; | |
620 | ||
621 | ConstraintOrRegister::Constraint(constraint) | |
622 | } | |
623 | ||
624 | /// Type to use for outputs that are discarded. It doesn't really matter what | |
625 | /// the type is, as long as it is valid for the constraint code. | |
626 | fn dummy_output_type<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, reg: InlineAsmRegClass) -> Type<'gcc> { | |
627 | match reg { | |
628 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => cx.type_i32(), | |
629 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => unimplemented!(), | |
630 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) | |
631 | | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => { | |
632 | unimplemented!() | |
633 | } | |
a2a8927a | 634 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg)=> cx.type_i32(), |
c295e0f8 XL |
635 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg) |
636 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16) => cx.type_f32(), | |
637 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg) | |
638 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16) | |
639 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8) => cx.type_f64(), | |
640 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg) | |
641 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8) | |
642 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4) => { | |
643 | unimplemented!() | |
644 | } | |
a2a8927a | 645 | InlineAsmRegClass::Avr(_) => unimplemented!(), |
c295e0f8 XL |
646 | InlineAsmRegClass::Bpf(_) => unimplemented!(), |
647 | InlineAsmRegClass::Hexagon(HexagonInlineAsmRegClass::reg) => cx.type_i32(), | |
648 | InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg) => cx.type_i32(), | |
649 | InlineAsmRegClass::Mips(MipsInlineAsmRegClass::freg) => cx.type_f32(), | |
5099ac24 | 650 | InlineAsmRegClass::Msp430(_) => unimplemented!(), |
c295e0f8 XL |
651 | InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg16) => cx.type_i16(), |
652 | InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg32) => cx.type_i32(), | |
653 | InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg64) => cx.type_i64(), | |
654 | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg) => cx.type_i32(), | |
655 | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => cx.type_i32(), | |
656 | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::freg) => cx.type_f64(), | |
657 | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::cr) | |
658 | | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::xer) => { | |
659 | unreachable!("clobber-only") | |
660 | }, | |
661 | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) => cx.type_i32(), | |
662 | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => cx.type_f32(), | |
663 | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::vreg) => cx.type_f32(), | |
664 | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg) | |
665 | | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => cx.type_i32(), | |
666 | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_byte) => cx.type_i8(), | |
667 | InlineAsmRegClass::X86(X86InlineAsmRegClass::mmx_reg) => unimplemented!(), | |
668 | InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg) | |
669 | | InlineAsmRegClass::X86(X86InlineAsmRegClass::ymm_reg) | |
670 | | InlineAsmRegClass::X86(X86InlineAsmRegClass::zmm_reg) => cx.type_f32(), | |
671 | InlineAsmRegClass::X86(X86InlineAsmRegClass::x87_reg) => unimplemented!(), | |
672 | InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => cx.type_i16(), | |
04454e1e | 673 | InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg0) => cx.type_i16(), |
923072b8 | 674 | InlineAsmRegClass::X86(X86InlineAsmRegClass::tmm_reg) => unimplemented!(), |
c295e0f8 XL |
675 | InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => cx.type_i32(), |
676 | InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { | |
677 | bug!("LLVM backend does not support SPIR-V") | |
678 | }, | |
679 | InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg) => cx.type_i32(), | |
680 | InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => cx.type_f64(), | |
681 | InlineAsmRegClass::Err => unreachable!(), | |
682 | } | |
683 | } | |
684 | ||
04454e1e FG |
685 | impl<'gcc, 'tcx> AsmMethods<'tcx> for CodegenCx<'gcc, 'tcx> { |
686 | fn codegen_global_asm(&self, template: &[InlineAsmTemplatePiece], operands: &[GlobalAsmOperandRef<'tcx>], options: InlineAsmOptions, _line_spans: &[Span]) { | |
c295e0f8 XL |
687 | let asm_arch = self.tcx.sess.asm_arch.unwrap(); |
688 | ||
689 | // Default to Intel syntax on x86 | |
923072b8 FG |
690 | let att_dialect = matches!(asm_arch, InlineAsmArch::X86 | InlineAsmArch::X86_64) |
691 | && options.contains(InlineAsmOptions::ATT_SYNTAX); | |
c295e0f8 XL |
692 | |
693 | // Build the template string | |
694 | let mut template_str = String::new(); | |
695 | for piece in template { | |
696 | match *piece { | |
697 | InlineAsmTemplatePiece::String(ref string) => { | |
698 | for line in string.lines() { | |
699 | // NOTE: gcc does not allow inline comment, so remove them. | |
700 | let line = | |
701 | if let Some(index) = line.rfind("//") { | |
702 | &line[..index] | |
703 | } | |
704 | else { | |
705 | line | |
706 | }; | |
707 | template_str.push_str(line); | |
708 | template_str.push('\n'); | |
709 | } | |
710 | }, | |
711 | InlineAsmTemplatePiece::Placeholder { operand_idx, modifier: _, span: _ } => { | |
712 | match operands[operand_idx] { | |
713 | GlobalAsmOperandRef::Const { ref string } => { | |
714 | // Const operands get injected directly into the | |
715 | // template. Note that we don't need to escape % | |
716 | // here unlike normal inline assembly. | |
717 | template_str.push_str(string); | |
718 | } | |
04454e1e FG |
719 | |
720 | GlobalAsmOperandRef::SymFn { instance } => { | |
721 | // TODO(@Amanieu): Additional mangling is needed on | |
722 | // some targets to add a leading underscore (Mach-O) | |
723 | // or byte count suffixes (x86 Windows). | |
724 | let name = self.tcx.symbol_name(instance).name; | |
725 | template_str.push_str(name); | |
726 | } | |
727 | ||
728 | GlobalAsmOperandRef::SymStatic { def_id } => { | |
729 | // TODO(@Amanieu): Additional mangling is needed on | |
730 | // some targets to add a leading underscore (Mach-O). | |
731 | let instance = Instance::mono(self.tcx, def_id); | |
732 | let name = self.tcx.symbol_name(instance).name; | |
733 | template_str.push_str(name); | |
734 | } | |
c295e0f8 XL |
735 | } |
736 | } | |
737 | } | |
738 | } | |
739 | ||
740 | let template_str = | |
923072b8 FG |
741 | if att_dialect { |
742 | format!(".att_syntax\n\t{}\n\t.intel_syntax noprefix", template_str) | |
c295e0f8 XL |
743 | } |
744 | else { | |
923072b8 | 745 | template_str |
c295e0f8 XL |
746 | }; |
747 | // NOTE: seems like gcc will put the asm in the wrong section, so set it to .text manually. | |
748 | let template_str = format!(".pushsection .text\n{}\n.popsection", template_str); | |
749 | self.context.add_top_level_asm(None, &template_str); | |
750 | } | |
751 | } | |
752 | ||
753 | fn modifier_to_gcc(arch: InlineAsmArch, reg: InlineAsmRegClass, modifier: Option<char>) -> Option<char> { | |
754 | match reg { | |
755 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => modifier, | |
756 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => modifier, | |
757 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) | |
758 | | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => { | |
759 | unimplemented!() | |
760 | } | |
a2a8927a | 761 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) => unimplemented!(), |
c295e0f8 XL |
762 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg) |
763 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16) => unimplemented!(), | |
764 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg) | |
765 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16) | |
766 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8) => unimplemented!(), | |
767 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg) | |
768 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8) | |
769 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4) => { | |
770 | unimplemented!() | |
771 | } | |
a2a8927a | 772 | InlineAsmRegClass::Avr(_) => unimplemented!(), |
c295e0f8 XL |
773 | InlineAsmRegClass::Bpf(_) => unimplemented!(), |
774 | InlineAsmRegClass::Hexagon(_) => unimplemented!(), | |
775 | InlineAsmRegClass::Mips(_) => unimplemented!(), | |
5099ac24 | 776 | InlineAsmRegClass::Msp430(_) => unimplemented!(), |
c295e0f8 XL |
777 | InlineAsmRegClass::Nvptx(_) => unimplemented!(), |
778 | InlineAsmRegClass::PowerPC(_) => unimplemented!(), | |
779 | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) | |
780 | | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => unimplemented!(), | |
781 | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::vreg) => unimplemented!(), | |
782 | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg) | |
783 | | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => match modifier { | |
784 | None => if arch == InlineAsmArch::X86_64 { Some('q') } else { Some('k') }, | |
785 | Some('l') => Some('b'), | |
786 | Some('h') => Some('h'), | |
787 | Some('x') => Some('w'), | |
788 | Some('e') => Some('k'), | |
789 | Some('r') => Some('q'), | |
790 | _ => unreachable!(), | |
791 | }, | |
792 | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_byte) => None, | |
793 | InlineAsmRegClass::X86(reg @ X86InlineAsmRegClass::xmm_reg) | |
794 | | InlineAsmRegClass::X86(reg @ X86InlineAsmRegClass::ymm_reg) | |
795 | | InlineAsmRegClass::X86(reg @ X86InlineAsmRegClass::zmm_reg) => match (reg, modifier) { | |
796 | (X86InlineAsmRegClass::xmm_reg, None) => Some('x'), | |
797 | (X86InlineAsmRegClass::ymm_reg, None) => Some('t'), | |
798 | (X86InlineAsmRegClass::zmm_reg, None) => Some('g'), | |
799 | (_, Some('x')) => Some('x'), | |
800 | (_, Some('y')) => Some('t'), | |
801 | (_, Some('z')) => Some('g'), | |
802 | _ => unreachable!(), | |
803 | }, | |
804 | InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => None, | |
04454e1e | 805 | InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg0) => None, |
923072b8 | 806 | InlineAsmRegClass::X86(X86InlineAsmRegClass::x87_reg | X86InlineAsmRegClass::mmx_reg | X86InlineAsmRegClass::tmm_reg) => { |
c295e0f8 XL |
807 | unreachable!("clobber-only") |
808 | } | |
809 | InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => unimplemented!(), | |
810 | InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { | |
811 | bug!("LLVM backend does not support SPIR-V") | |
812 | }, | |
813 | InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg) => unimplemented!(), | |
814 | InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => unimplemented!(), | |
815 | InlineAsmRegClass::Err => unreachable!(), | |
816 | } | |
817 | } |