]>
Commit | Line | Data |
---|---|---|
dfeec247 | 1 | use crate::builder::Builder; |
9fa01778 | 2 | use crate::context::CodegenCx; |
dfeec247 | 3 | use crate::llvm; |
f9f354fc | 4 | use crate::type_::Type; |
9fa01778 | 5 | use crate::type_of::LayoutLlvmExt; |
9fa01778 | 6 | use crate::value::Value; |
54a0048b | 7 | |
3dfed10e XL |
8 | use rustc_ast::LlvmAsmDialect; |
9 | use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece}; | |
a1dfa0c6 | 10 | use rustc_codegen_ssa::mir::operand::OperandValue; |
dfeec247 XL |
11 | use rustc_codegen_ssa::mir::place::PlaceRef; |
12 | use rustc_codegen_ssa::traits::*; | |
f9f354fc | 13 | use rustc_data_structures::fx::FxHashMap; |
dfeec247 | 14 | use rustc_hir as hir; |
f9f354fc XL |
15 | use rustc_middle::span_bug; |
16 | use rustc_middle::ty::layout::TyAndLayout; | |
17 | use rustc_span::{Pos, Span}; | |
18 | use rustc_target::abi::*; | |
19 | use rustc_target::asm::*; | |
32a655c1 | 20 | |
dfeec247 | 21 | use libc::{c_char, c_uint}; |
3dfed10e | 22 | use tracing::debug; |
a1dfa0c6 XL |
23 | |
24 | impl AsmBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> { | |
ba9703b0 | 25 | fn codegen_llvm_inline_asm( |
a1dfa0c6 | 26 | &mut self, |
ba9703b0 | 27 | ia: &hir::LlvmInlineAsmInner, |
a1dfa0c6 | 28 | outputs: Vec<PlaceRef<'tcx, &'ll Value>>, |
e1599b0c XL |
29 | mut inputs: Vec<&'ll Value>, |
30 | span: Span, | |
a1dfa0c6 XL |
31 | ) -> bool { |
32 | let mut ext_constraints = vec![]; | |
33 | let mut output_types = vec![]; | |
34 | ||
35 | // Prepare the output operands | |
36 | let mut indirect_outputs = vec![]; | |
37 | for (i, (out, &place)) in ia.outputs.iter().zip(&outputs).enumerate() { | |
38 | if out.is_rw { | |
74b04a01 XL |
39 | let operand = self.load_operand(place); |
40 | if let OperandValue::Immediate(_) = operand.val { | |
41 | inputs.push(operand.immediate()); | |
42 | } | |
a1dfa0c6 XL |
43 | ext_constraints.push(i.to_string()); |
44 | } | |
45 | if out.is_indirect { | |
74b04a01 XL |
46 | let operand = self.load_operand(place); |
47 | if let OperandValue::Immediate(_) = operand.val { | |
48 | indirect_outputs.push(operand.immediate()); | |
49 | } | |
a1dfa0c6 | 50 | } else { |
f9f354fc | 51 | output_types.push(place.layout.llvm_type(self.cx)); |
a1dfa0c6 | 52 | } |
54a0048b | 53 | } |
a1dfa0c6 XL |
54 | if !indirect_outputs.is_empty() { |
55 | indirect_outputs.extend_from_slice(&inputs); | |
56 | inputs = indirect_outputs; | |
54a0048b | 57 | } |
54a0048b | 58 | |
dfeec247 | 59 | let clobbers = ia.clobbers.iter().map(|s| format!("~{{{}}}", &s)); |
a1dfa0c6 XL |
60 | |
61 | // Default per-arch clobbers | |
62 | // Basically what clang does | |
63 | let arch_clobbers = match &self.sess().target.target.arch[..] { | |
dfeec247 | 64 | "x86" | "x86_64" => vec!["~{dirflag}", "~{fpsr}", "~{flags}"], |
a1dfa0c6 | 65 | "mips" | "mips64" => vec!["~{$1}"], |
dfeec247 | 66 | _ => Vec::new(), |
a1dfa0c6 XL |
67 | }; |
68 | ||
dfeec247 XL |
69 | let all_constraints = ia |
70 | .outputs | |
71 | .iter() | |
72 | .map(|out| out.constraint.to_string()) | |
73 | .chain(ia.inputs.iter().map(|s| s.to_string())) | |
74 | .chain(ext_constraints) | |
75 | .chain(clobbers) | |
74b04a01 | 76 | .chain(arch_clobbers.iter().map(|s| (*s).to_string())) |
dfeec247 XL |
77 | .collect::<Vec<String>>() |
78 | .join(","); | |
a1dfa0c6 XL |
79 | |
80 | debug!("Asm Constraints: {}", &all_constraints); | |
81 | ||
82 | // Depending on how many outputs we have, the return type is different | |
83 | let num_outputs = output_types.len(); | |
84 | let output_type = match num_outputs { | |
85 | 0 => self.type_void(), | |
86 | 1 => output_types[0], | |
dfeec247 | 87 | _ => self.type_struct(&output_types, false), |
a1dfa0c6 XL |
88 | }; |
89 | ||
ba9703b0 | 90 | let asm = ia.asm.as_str(); |
532ac7d7 XL |
91 | let r = inline_asm_call( |
92 | self, | |
a1dfa0c6 | 93 | &asm, |
ba9703b0 | 94 | &all_constraints, |
a1dfa0c6 XL |
95 | &inputs, |
96 | output_type, | |
97 | ia.volatile, | |
98 | ia.alignstack, | |
dfeec247 | 99 | ia.dialect, |
f9f354fc | 100 | &[span], |
a1dfa0c6 XL |
101 | ); |
102 | if r.is_none() { | |
103 | return false; | |
104 | } | |
105 | let r = r.unwrap(); | |
54a0048b | 106 | |
a1dfa0c6 XL |
107 | // Again, based on how many outputs we have |
108 | let outputs = ia.outputs.iter().zip(&outputs).filter(|&(ref o, _)| !o.is_indirect); | |
109 | for (i, (_, &place)) in outputs.enumerate() { | |
110 | let v = if num_outputs == 1 { r } else { self.extract_value(r, i as u64) }; | |
111 | OperandValue::Immediate(v).store(self, place); | |
112 | } | |
54a0048b | 113 | |
f9f354fc XL |
114 | true |
115 | } | |
54a0048b | 116 | |
f9f354fc XL |
117 | fn codegen_inline_asm( |
118 | &mut self, | |
119 | template: &[InlineAsmTemplatePiece], | |
120 | operands: &[InlineAsmOperandRef<'tcx, Self>], | |
121 | options: InlineAsmOptions, | |
122 | line_spans: &[Span], | |
123 | ) { | |
124 | let asm_arch = self.tcx.sess.asm_arch.unwrap(); | |
54a0048b | 125 | |
f9f354fc XL |
126 | // Collect the types of output operands |
127 | let mut constraints = vec![]; | |
128 | let mut output_types = vec![]; | |
129 | let mut op_idx = FxHashMap::default(); | |
130 | for (idx, op) in operands.iter().enumerate() { | |
131 | match *op { | |
132 | InlineAsmOperandRef::Out { reg, late, place } => { | |
3dfed10e XL |
133 | let mut layout = None; |
134 | let ty = if let Some(ref place) = place { | |
135 | layout = Some(&place.layout); | |
f9f354fc XL |
136 | llvm_fixup_output_type(self.cx, reg.reg_class(), &place.layout) |
137 | } else { | |
138 | // If the output is discarded, we don't really care what | |
139 | // type is used. We're just using this to tell LLVM to | |
140 | // reserve the register. | |
141 | dummy_output_type(self.cx, reg.reg_class()) | |
142 | }; | |
143 | output_types.push(ty); | |
144 | op_idx.insert(idx, constraints.len()); | |
145 | let prefix = if late { "=" } else { "=&" }; | |
3dfed10e | 146 | constraints.push(format!("{}{}", prefix, reg_to_llvm(reg, layout))); |
f9f354fc XL |
147 | } |
148 | InlineAsmOperandRef::InOut { reg, late, in_value, out_place } => { | |
3dfed10e XL |
149 | let layout = if let Some(ref out_place) = out_place { |
150 | &out_place.layout | |
f9f354fc XL |
151 | } else { |
152 | // LLVM required tied operands to have the same type, | |
153 | // so we just use the type of the input. | |
3dfed10e | 154 | &in_value.layout |
f9f354fc | 155 | }; |
3dfed10e | 156 | let ty = llvm_fixup_output_type(self.cx, reg.reg_class(), layout); |
f9f354fc XL |
157 | output_types.push(ty); |
158 | op_idx.insert(idx, constraints.len()); | |
159 | let prefix = if late { "=" } else { "=&" }; | |
3dfed10e | 160 | constraints.push(format!("{}{}", prefix, reg_to_llvm(reg, Some(layout)))); |
f9f354fc XL |
161 | } |
162 | _ => {} | |
163 | } | |
a1dfa0c6 | 164 | } |
0bf4aa26 | 165 | |
f9f354fc XL |
166 | // Collect input operands |
167 | let mut inputs = vec![]; | |
168 | for (idx, op) in operands.iter().enumerate() { | |
169 | match *op { | |
170 | InlineAsmOperandRef::In { reg, value } => { | |
3dfed10e | 171 | let llval = |
f9f354fc | 172 | llvm_fixup_input(self, value.immediate(), reg.reg_class(), &value.layout); |
3dfed10e | 173 | inputs.push(llval); |
f9f354fc | 174 | op_idx.insert(idx, constraints.len()); |
3dfed10e | 175 | constraints.push(reg_to_llvm(reg, Some(&value.layout))); |
f9f354fc XL |
176 | } |
177 | InlineAsmOperandRef::InOut { reg, late: _, in_value, out_place: _ } => { | |
178 | let value = llvm_fixup_input( | |
179 | self, | |
180 | in_value.immediate(), | |
181 | reg.reg_class(), | |
182 | &in_value.layout, | |
183 | ); | |
184 | inputs.push(value); | |
185 | constraints.push(format!("{}", op_idx[&idx])); | |
186 | } | |
187 | InlineAsmOperandRef::SymFn { instance } => { | |
188 | inputs.push(self.cx.get_fn(instance)); | |
189 | op_idx.insert(idx, constraints.len()); | |
190 | constraints.push("s".to_string()); | |
191 | } | |
192 | InlineAsmOperandRef::SymStatic { def_id } => { | |
193 | inputs.push(self.cx.get_static(def_id)); | |
194 | op_idx.insert(idx, constraints.len()); | |
195 | constraints.push("s".to_string()); | |
196 | } | |
197 | _ => {} | |
198 | } | |
199 | } | |
200 | ||
201 | // Build the template string | |
202 | let mut template_str = String::new(); | |
203 | for piece in template { | |
204 | match *piece { | |
205 | InlineAsmTemplatePiece::String(ref s) => { | |
206 | if s.contains('$') { | |
207 | for c in s.chars() { | |
208 | if c == '$' { | |
209 | template_str.push_str("$$"); | |
210 | } else { | |
211 | template_str.push(c); | |
212 | } | |
213 | } | |
214 | } else { | |
215 | template_str.push_str(s) | |
216 | } | |
217 | } | |
218 | InlineAsmTemplatePiece::Placeholder { operand_idx, modifier, span: _ } => { | |
219 | match operands[operand_idx] { | |
220 | InlineAsmOperandRef::In { reg, .. } | |
221 | | InlineAsmOperandRef::Out { reg, .. } | |
222 | | InlineAsmOperandRef::InOut { reg, .. } => { | |
223 | let modifier = modifier_to_llvm(asm_arch, reg.reg_class(), modifier); | |
224 | if let Some(modifier) = modifier { | |
225 | template_str.push_str(&format!( | |
226 | "${{{}:{}}}", | |
227 | op_idx[&operand_idx], modifier | |
228 | )); | |
229 | } else { | |
230 | template_str.push_str(&format!("${{{}}}", op_idx[&operand_idx])); | |
231 | } | |
232 | } | |
233 | InlineAsmOperandRef::Const { ref string } => { | |
234 | // Const operands get injected directly into the template | |
235 | template_str.push_str(string); | |
236 | } | |
237 | InlineAsmOperandRef::SymFn { .. } | |
238 | | InlineAsmOperandRef::SymStatic { .. } => { | |
239 | // Only emit the raw symbol name | |
240 | template_str.push_str(&format!("${{{}:c}}", op_idx[&operand_idx])); | |
241 | } | |
242 | } | |
243 | } | |
244 | } | |
245 | } | |
246 | ||
247 | if !options.contains(InlineAsmOptions::PRESERVES_FLAGS) { | |
248 | match asm_arch { | |
249 | InlineAsmArch::AArch64 | InlineAsmArch::Arm => { | |
250 | constraints.push("~{cc}".to_string()); | |
251 | } | |
252 | InlineAsmArch::X86 | InlineAsmArch::X86_64 => { | |
253 | constraints.extend_from_slice(&[ | |
254 | "~{dirflag}".to_string(), | |
255 | "~{fpsr}".to_string(), | |
256 | "~{flags}".to_string(), | |
257 | ]); | |
258 | } | |
259 | InlineAsmArch::RiscV32 | InlineAsmArch::RiscV64 => {} | |
260 | InlineAsmArch::Nvptx64 => {} | |
f035d41b | 261 | InlineAsmArch::Hexagon => {} |
f9f354fc XL |
262 | } |
263 | } | |
264 | if !options.contains(InlineAsmOptions::NOMEM) { | |
265 | // This is actually ignored by LLVM, but it's probably best to keep | |
266 | // it just in case. LLVM instead uses the ReadOnly/ReadNone | |
267 | // attributes on the call instruction to optimize. | |
268 | constraints.push("~{memory}".to_string()); | |
269 | } | |
270 | let volatile = !options.contains(InlineAsmOptions::PURE); | |
271 | let alignstack = !options.contains(InlineAsmOptions::NOSTACK); | |
272 | let output_type = match &output_types[..] { | |
273 | [] => self.type_void(), | |
274 | [ty] => ty, | |
275 | tys => self.type_struct(&tys, false), | |
276 | }; | |
277 | let dialect = match asm_arch { | |
278 | InlineAsmArch::X86 | InlineAsmArch::X86_64 | |
279 | if !options.contains(InlineAsmOptions::ATT_SYNTAX) => | |
280 | { | |
281 | LlvmAsmDialect::Intel | |
282 | } | |
283 | _ => LlvmAsmDialect::Att, | |
284 | }; | |
285 | let result = inline_asm_call( | |
286 | self, | |
287 | &template_str, | |
288 | &constraints.join(","), | |
289 | &inputs, | |
290 | output_type, | |
291 | volatile, | |
292 | alignstack, | |
293 | dialect, | |
294 | line_spans, | |
295 | ) | |
296 | .unwrap_or_else(|| span_bug!(line_spans[0], "LLVM asm constraint validation failed")); | |
297 | ||
298 | if options.contains(InlineAsmOptions::PURE) { | |
299 | if options.contains(InlineAsmOptions::NOMEM) { | |
300 | llvm::Attribute::ReadNone.apply_callsite(llvm::AttributePlace::Function, result); | |
301 | } else if options.contains(InlineAsmOptions::READONLY) { | |
302 | llvm::Attribute::ReadOnly.apply_callsite(llvm::AttributePlace::Function, result); | |
303 | } | |
304 | } else { | |
305 | if options.contains(InlineAsmOptions::NOMEM) { | |
306 | llvm::Attribute::InaccessibleMemOnly | |
307 | .apply_callsite(llvm::AttributePlace::Function, result); | |
308 | } else { | |
309 | // LLVM doesn't have an attribute to represent ReadOnly + SideEffect | |
310 | } | |
311 | } | |
312 | ||
313 | // Write results to outputs | |
314 | for (idx, op) in operands.iter().enumerate() { | |
315 | if let InlineAsmOperandRef::Out { reg, place: Some(place), .. } | |
316 | | InlineAsmOperandRef::InOut { reg, out_place: Some(place), .. } = *op | |
317 | { | |
318 | let value = if output_types.len() == 1 { | |
319 | result | |
320 | } else { | |
321 | self.extract_value(result, op_idx[&idx] as u64) | |
322 | }; | |
323 | let value = llvm_fixup_output(self, value, reg.reg_class(), &place.layout); | |
324 | OperandValue::Immediate(value).store(self, place); | |
325 | } | |
326 | } | |
a1dfa0c6 | 327 | } |
54a0048b | 328 | } |
cc61c64b | 329 | |
dc9dc135 | 330 | impl AsmMethods for CodegenCx<'ll, 'tcx> { |
a1dfa0c6 | 331 | fn codegen_global_asm(&self, ga: &hir::GlobalAsm) { |
ba9703b0 | 332 | let asm = ga.asm.as_str(); |
a1dfa0c6 | 333 | unsafe { |
ba9703b0 | 334 | llvm::LLVMRustAppendModuleInlineAsm(self.llmod, asm.as_ptr().cast(), asm.len()); |
a1dfa0c6 | 335 | } |
cc61c64b XL |
336 | } |
337 | } | |
532ac7d7 XL |
338 | |
339 | fn inline_asm_call( | |
340 | bx: &mut Builder<'a, 'll, 'tcx>, | |
ba9703b0 XL |
341 | asm: &str, |
342 | cons: &str, | |
532ac7d7 XL |
343 | inputs: &[&'ll Value], |
344 | output: &'ll llvm::Type, | |
345 | volatile: bool, | |
346 | alignstack: bool, | |
f9f354fc XL |
347 | dia: LlvmAsmDialect, |
348 | line_spans: &[Span], | |
532ac7d7 | 349 | ) -> Option<&'ll Value> { |
dfeec247 XL |
350 | let volatile = if volatile { llvm::True } else { llvm::False }; |
351 | let alignstack = if alignstack { llvm::True } else { llvm::False }; | |
352 | ||
353 | let argtys = inputs | |
354 | .iter() | |
355 | .map(|v| { | |
356 | debug!("Asm Input Type: {:?}", *v); | |
357 | bx.cx.val_ty(*v) | |
358 | }) | |
359 | .collect::<Vec<_>>(); | |
532ac7d7 XL |
360 | |
361 | debug!("Asm Output Type: {:?}", output); | |
362 | let fty = bx.cx.type_func(&argtys[..], output); | |
363 | unsafe { | |
364 | // Ask LLVM to verify that the constraints are well-formed. | |
ba9703b0 | 365 | let constraints_ok = llvm::LLVMRustInlineAsmVerify(fty, cons.as_ptr().cast(), cons.len()); |
416331ca | 366 | debug!("constraint verification result: {:?}", constraints_ok); |
532ac7d7 XL |
367 | if constraints_ok { |
368 | let v = llvm::LLVMRustInlineAsm( | |
369 | fty, | |
ba9703b0 XL |
370 | asm.as_ptr().cast(), |
371 | asm.len(), | |
372 | cons.as_ptr().cast(), | |
373 | cons.len(), | |
532ac7d7 XL |
374 | volatile, |
375 | alignstack, | |
376 | llvm::AsmDialect::from_generic(dia), | |
377 | ); | |
f9f354fc XL |
378 | let call = bx.call(v, inputs, None); |
379 | ||
380 | // Store mark in a metadata node so we can map LLVM errors | |
381 | // back to source locations. See #17552. | |
382 | let key = "srcloc"; | |
383 | let kind = llvm::LLVMGetMDKindIDInContext( | |
384 | bx.llcx, | |
385 | key.as_ptr() as *const c_char, | |
386 | key.len() as c_uint, | |
387 | ); | |
388 | ||
389 | // srcloc contains one integer for each line of assembly code. | |
390 | // Unfortunately this isn't enough to encode a full span so instead | |
391 | // we just encode the start position of each line. | |
392 | // FIXME: Figure out a way to pass the entire line spans. | |
393 | let mut srcloc = vec![]; | |
394 | if dia == LlvmAsmDialect::Intel && line_spans.len() > 1 { | |
395 | // LLVM inserts an extra line to add the ".intel_syntax", so add | |
396 | // a dummy srcloc entry for it. | |
397 | // | |
398 | // Don't do this if we only have 1 line span since that may be | |
399 | // due to the asm template string coming from a macro. LLVM will | |
400 | // default to the first srcloc for lines that don't have an | |
401 | // associated srcloc. | |
402 | srcloc.push(bx.const_i32(0)); | |
403 | } | |
404 | srcloc.extend(line_spans.iter().map(|span| bx.const_i32(span.lo().to_u32() as i32))); | |
405 | let md = llvm::LLVMMDNodeInContext(bx.llcx, srcloc.as_ptr(), srcloc.len() as u32); | |
406 | llvm::LLVMSetMetadata(call, kind, md); | |
407 | ||
408 | Some(call) | |
532ac7d7 XL |
409 | } else { |
410 | // LLVM has detected an issue with our constraints, bail out | |
411 | None | |
412 | } | |
413 | } | |
414 | } | |
f9f354fc | 415 | |
3dfed10e XL |
416 | /// If the register is an xmm/ymm/zmm register then return its index. |
417 | fn xmm_reg_index(reg: InlineAsmReg) -> Option<u32> { | |
418 | match reg { | |
419 | InlineAsmReg::X86(reg) | |
420 | if reg as u32 >= X86InlineAsmReg::xmm0 as u32 | |
421 | && reg as u32 <= X86InlineAsmReg::xmm15 as u32 => | |
422 | { | |
423 | Some(reg as u32 - X86InlineAsmReg::xmm0 as u32) | |
424 | } | |
425 | InlineAsmReg::X86(reg) | |
426 | if reg as u32 >= X86InlineAsmReg::ymm0 as u32 | |
427 | && reg as u32 <= X86InlineAsmReg::ymm15 as u32 => | |
428 | { | |
429 | Some(reg as u32 - X86InlineAsmReg::ymm0 as u32) | |
430 | } | |
431 | InlineAsmReg::X86(reg) | |
432 | if reg as u32 >= X86InlineAsmReg::zmm0 as u32 | |
433 | && reg as u32 <= X86InlineAsmReg::zmm31 as u32 => | |
434 | { | |
435 | Some(reg as u32 - X86InlineAsmReg::zmm0 as u32) | |
436 | } | |
437 | _ => None, | |
438 | } | |
439 | } | |
440 | ||
441 | /// If the register is an AArch64 vector register then return its index. | |
442 | fn a64_vreg_index(reg: InlineAsmReg) -> Option<u32> { | |
443 | match reg { | |
444 | InlineAsmReg::AArch64(reg) | |
445 | if reg as u32 >= AArch64InlineAsmReg::v0 as u32 | |
446 | && reg as u32 <= AArch64InlineAsmReg::v31 as u32 => | |
447 | { | |
448 | Some(reg as u32 - AArch64InlineAsmReg::v0 as u32) | |
449 | } | |
450 | _ => None, | |
451 | } | |
452 | } | |
453 | ||
f9f354fc | 454 | /// Converts a register class to an LLVM constraint code. |
3dfed10e | 455 | fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'tcx>>) -> String { |
f9f354fc | 456 | match reg { |
3dfed10e XL |
457 | // For vector registers LLVM wants the register name to match the type size. |
458 | InlineAsmRegOrRegClass::Reg(reg) => { | |
459 | if let Some(idx) = xmm_reg_index(reg) { | |
460 | let class = if let Some(layout) = layout { | |
461 | match layout.size.bytes() { | |
462 | 64 => 'z', | |
463 | 32 => 'y', | |
464 | _ => 'x', | |
465 | } | |
466 | } else { | |
467 | // We use f32 as the type for discarded outputs | |
468 | 'x' | |
469 | }; | |
470 | format!("{{{}mm{}}}", class, idx) | |
471 | } else if let Some(idx) = a64_vreg_index(reg) { | |
472 | let class = if let Some(layout) = layout { | |
473 | match layout.size.bytes() { | |
474 | 16 => 'q', | |
475 | 8 => 'd', | |
476 | 4 => 's', | |
477 | 2 => 'h', | |
478 | 1 => 'd', // We fixup i8 to i8x8 | |
479 | _ => unreachable!(), | |
480 | } | |
481 | } else { | |
482 | // We use i64x2 as the type for discarded outputs | |
483 | 'q' | |
484 | }; | |
485 | format!("{{{}{}}}", class, idx) | |
486 | } else if reg == InlineAsmReg::AArch64(AArch64InlineAsmReg::x30) { | |
487 | // LLVM doesn't recognize x30 | |
488 | "lr".to_string() | |
489 | } else { | |
490 | format!("{{{}}}", reg.name()) | |
491 | } | |
492 | } | |
f9f354fc XL |
493 | InlineAsmRegOrRegClass::RegClass(reg) => match reg { |
494 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => "r", | |
495 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) => "w", | |
496 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => "x", | |
497 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) => "r", | |
498 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg_thumb) => "l", | |
499 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg) | |
500 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16) | |
501 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8) => "t", | |
502 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16) | |
503 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8) | |
504 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4) => "x", | |
505 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg) | |
506 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg) => "w", | |
f035d41b | 507 | InlineAsmRegClass::Hexagon(HexagonInlineAsmRegClass::reg) => "r", |
f9f354fc XL |
508 | InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg16) => "h", |
509 | InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg32) => "r", | |
510 | InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg64) => "l", | |
511 | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) => "r", | |
512 | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => "f", | |
513 | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg) => "r", | |
514 | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => "Q", | |
515 | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_byte) => "q", | |
516 | InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg) | |
517 | | InlineAsmRegClass::X86(X86InlineAsmRegClass::ymm_reg) => "x", | |
518 | InlineAsmRegClass::X86(X86InlineAsmRegClass::zmm_reg) => "v", | |
519 | InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => "^Yk", | |
520 | } | |
521 | .to_string(), | |
522 | } | |
523 | } | |
524 | ||
525 | /// Converts a modifier into LLVM's equivalent modifier. | |
526 | fn modifier_to_llvm( | |
527 | arch: InlineAsmArch, | |
528 | reg: InlineAsmRegClass, | |
529 | modifier: Option<char>, | |
530 | ) -> Option<char> { | |
531 | match reg { | |
532 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => modifier, | |
533 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) | |
534 | | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => { | |
535 | if modifier == Some('v') { None } else { modifier } | |
536 | } | |
537 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) | |
538 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg_thumb) => None, | |
539 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg) | |
540 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16) => None, | |
541 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg) | |
542 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16) | |
543 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8) => Some('P'), | |
544 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg) | |
545 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8) | |
546 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4) => { | |
547 | if modifier.is_none() { | |
548 | Some('q') | |
549 | } else { | |
550 | modifier | |
551 | } | |
552 | } | |
f035d41b | 553 | InlineAsmRegClass::Hexagon(_) => None, |
f9f354fc XL |
554 | InlineAsmRegClass::Nvptx(_) => None, |
555 | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) | |
556 | | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => None, | |
557 | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg) | |
558 | | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => match modifier { | |
559 | None if arch == InlineAsmArch::X86_64 => Some('q'), | |
560 | None => Some('k'), | |
561 | Some('l') => Some('b'), | |
562 | Some('h') => Some('h'), | |
563 | Some('x') => Some('w'), | |
564 | Some('e') => Some('k'), | |
565 | Some('r') => Some('q'), | |
566 | _ => unreachable!(), | |
567 | }, | |
568 | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_byte) => None, | |
569 | InlineAsmRegClass::X86(reg @ X86InlineAsmRegClass::xmm_reg) | |
570 | | InlineAsmRegClass::X86(reg @ X86InlineAsmRegClass::ymm_reg) | |
571 | | InlineAsmRegClass::X86(reg @ X86InlineAsmRegClass::zmm_reg) => match (reg, modifier) { | |
572 | (X86InlineAsmRegClass::xmm_reg, None) => Some('x'), | |
573 | (X86InlineAsmRegClass::ymm_reg, None) => Some('t'), | |
574 | (X86InlineAsmRegClass::zmm_reg, None) => Some('g'), | |
575 | (_, Some('x')) => Some('x'), | |
576 | (_, Some('y')) => Some('t'), | |
577 | (_, Some('z')) => Some('g'), | |
578 | _ => unreachable!(), | |
579 | }, | |
580 | InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => None, | |
581 | } | |
582 | } | |
583 | ||
584 | /// Type to use for outputs that are discarded. It doesn't really matter what | |
585 | /// the type is, as long as it is valid for the constraint code. | |
586 | fn dummy_output_type(cx: &CodegenCx<'ll, 'tcx>, reg: InlineAsmRegClass) -> &'ll Type { | |
587 | match reg { | |
588 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => cx.type_i32(), | |
589 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) | |
590 | | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => { | |
591 | cx.type_vector(cx.type_i64(), 2) | |
592 | } | |
593 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) | |
594 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg_thumb) => cx.type_i32(), | |
595 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg) | |
596 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16) => cx.type_f32(), | |
597 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg) | |
598 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16) | |
599 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8) => cx.type_f64(), | |
600 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg) | |
601 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8) | |
602 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4) => { | |
603 | cx.type_vector(cx.type_i64(), 2) | |
604 | } | |
f035d41b | 605 | InlineAsmRegClass::Hexagon(HexagonInlineAsmRegClass::reg) => cx.type_i32(), |
f9f354fc XL |
606 | InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg16) => cx.type_i16(), |
607 | InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg32) => cx.type_i32(), | |
608 | InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg64) => cx.type_i64(), | |
609 | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) => cx.type_i32(), | |
610 | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => cx.type_f32(), | |
611 | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg) | |
612 | | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => cx.type_i32(), | |
613 | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_byte) => cx.type_i8(), | |
614 | InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg) | |
615 | | InlineAsmRegClass::X86(X86InlineAsmRegClass::ymm_reg) | |
616 | | InlineAsmRegClass::X86(X86InlineAsmRegClass::zmm_reg) => cx.type_f32(), | |
617 | InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => cx.type_i16(), | |
618 | } | |
619 | } | |
620 | ||
621 | /// Helper function to get the LLVM type for a Scalar. Pointers are returned as | |
622 | /// the equivalent integer type. | |
623 | fn llvm_asm_scalar_type(cx: &CodegenCx<'ll, 'tcx>, scalar: &Scalar) -> &'ll Type { | |
624 | match scalar.value { | |
625 | Primitive::Int(Integer::I8, _) => cx.type_i8(), | |
626 | Primitive::Int(Integer::I16, _) => cx.type_i16(), | |
627 | Primitive::Int(Integer::I32, _) => cx.type_i32(), | |
628 | Primitive::Int(Integer::I64, _) => cx.type_i64(), | |
629 | Primitive::F32 => cx.type_f32(), | |
630 | Primitive::F64 => cx.type_f64(), | |
631 | Primitive::Pointer => cx.type_isize(), | |
632 | _ => unreachable!(), | |
633 | } | |
634 | } | |
635 | ||
636 | /// Fix up an input value to work around LLVM bugs. | |
637 | fn llvm_fixup_input( | |
638 | bx: &mut Builder<'a, 'll, 'tcx>, | |
639 | mut value: &'ll Value, | |
640 | reg: InlineAsmRegClass, | |
641 | layout: &TyAndLayout<'tcx>, | |
642 | ) -> &'ll Value { | |
643 | match (reg, &layout.abi) { | |
644 | (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => { | |
645 | if let Primitive::Int(Integer::I8, _) = s.value { | |
646 | let vec_ty = bx.cx.type_vector(bx.cx.type_i8(), 8); | |
647 | bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0)) | |
648 | } else { | |
649 | value | |
650 | } | |
651 | } | |
652 | (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) => { | |
653 | let elem_ty = llvm_asm_scalar_type(bx.cx, s); | |
654 | let count = 16 / layout.size.bytes(); | |
655 | let vec_ty = bx.cx.type_vector(elem_ty, count); | |
656 | if let Primitive::Pointer = s.value { | |
657 | value = bx.ptrtoint(value, bx.cx.type_isize()); | |
658 | } | |
659 | bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0)) | |
660 | } | |
661 | ( | |
662 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), | |
663 | Abi::Vector { element, count }, | |
664 | ) if layout.size.bytes() == 8 => { | |
665 | let elem_ty = llvm_asm_scalar_type(bx.cx, element); | |
666 | let vec_ty = bx.cx.type_vector(elem_ty, *count); | |
667 | let indices: Vec<_> = (0..count * 2).map(|x| bx.const_i32(x as i32)).collect(); | |
668 | bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices)) | |
669 | } | |
670 | (InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd), Abi::Scalar(s)) | |
671 | if s.value == Primitive::F64 => | |
672 | { | |
673 | bx.bitcast(value, bx.cx.type_i64()) | |
674 | } | |
675 | ( | |
676 | InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg), | |
677 | Abi::Vector { .. }, | |
678 | ) if layout.size.bytes() == 64 => bx.bitcast(value, bx.cx.type_vector(bx.cx.type_f64(), 8)), | |
3dfed10e XL |
679 | ( |
680 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16), | |
681 | Abi::Scalar(s), | |
682 | ) => { | |
683 | if let Primitive::Int(Integer::I32, _) = s.value { | |
684 | bx.bitcast(value, bx.cx.type_f32()) | |
685 | } else { | |
686 | value | |
687 | } | |
688 | } | |
f9f354fc XL |
689 | ( |
690 | InlineAsmRegClass::Arm( | |
3dfed10e | 691 | ArmInlineAsmRegClass::dreg |
f9f354fc | 692 | | ArmInlineAsmRegClass::dreg_low8 |
3dfed10e | 693 | | ArmInlineAsmRegClass::dreg_low16, |
f9f354fc XL |
694 | ), |
695 | Abi::Scalar(s), | |
696 | ) => { | |
3dfed10e XL |
697 | if let Primitive::Int(Integer::I64, _) = s.value { |
698 | bx.bitcast(value, bx.cx.type_f64()) | |
f9f354fc XL |
699 | } else { |
700 | value | |
701 | } | |
702 | } | |
703 | _ => value, | |
704 | } | |
705 | } | |
706 | ||
707 | /// Fix up an output value to work around LLVM bugs. | |
708 | fn llvm_fixup_output( | |
709 | bx: &mut Builder<'a, 'll, 'tcx>, | |
710 | mut value: &'ll Value, | |
711 | reg: InlineAsmRegClass, | |
712 | layout: &TyAndLayout<'tcx>, | |
713 | ) -> &'ll Value { | |
714 | match (reg, &layout.abi) { | |
715 | (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => { | |
716 | if let Primitive::Int(Integer::I8, _) = s.value { | |
717 | bx.extract_element(value, bx.const_i32(0)) | |
718 | } else { | |
719 | value | |
720 | } | |
721 | } | |
722 | (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) => { | |
723 | value = bx.extract_element(value, bx.const_i32(0)); | |
724 | if let Primitive::Pointer = s.value { | |
725 | value = bx.inttoptr(value, layout.llvm_type(bx.cx)); | |
726 | } | |
727 | value | |
728 | } | |
729 | ( | |
730 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), | |
731 | Abi::Vector { element, count }, | |
732 | ) if layout.size.bytes() == 8 => { | |
733 | let elem_ty = llvm_asm_scalar_type(bx.cx, element); | |
734 | let vec_ty = bx.cx.type_vector(elem_ty, *count * 2); | |
735 | let indices: Vec<_> = (0..*count).map(|x| bx.const_i32(x as i32)).collect(); | |
736 | bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices)) | |
737 | } | |
738 | (InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd), Abi::Scalar(s)) | |
739 | if s.value == Primitive::F64 => | |
740 | { | |
741 | bx.bitcast(value, bx.cx.type_f64()) | |
742 | } | |
743 | ( | |
744 | InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg), | |
745 | Abi::Vector { .. }, | |
746 | ) if layout.size.bytes() == 64 => bx.bitcast(value, layout.llvm_type(bx.cx)), | |
3dfed10e XL |
747 | ( |
748 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16), | |
749 | Abi::Scalar(s), | |
750 | ) => { | |
751 | if let Primitive::Int(Integer::I32, _) = s.value { | |
752 | bx.bitcast(value, bx.cx.type_i32()) | |
753 | } else { | |
754 | value | |
755 | } | |
756 | } | |
f9f354fc XL |
757 | ( |
758 | InlineAsmRegClass::Arm( | |
3dfed10e | 759 | ArmInlineAsmRegClass::dreg |
f9f354fc | 760 | | ArmInlineAsmRegClass::dreg_low8 |
3dfed10e | 761 | | ArmInlineAsmRegClass::dreg_low16, |
f9f354fc XL |
762 | ), |
763 | Abi::Scalar(s), | |
764 | ) => { | |
3dfed10e XL |
765 | if let Primitive::Int(Integer::I64, _) = s.value { |
766 | bx.bitcast(value, bx.cx.type_i64()) | |
f9f354fc XL |
767 | } else { |
768 | value | |
769 | } | |
770 | } | |
771 | _ => value, | |
772 | } | |
773 | } | |
774 | ||
775 | /// Output type to use for llvm_fixup_output. | |
776 | fn llvm_fixup_output_type( | |
777 | cx: &CodegenCx<'ll, 'tcx>, | |
778 | reg: InlineAsmRegClass, | |
779 | layout: &TyAndLayout<'tcx>, | |
780 | ) -> &'ll Type { | |
781 | match (reg, &layout.abi) { | |
782 | (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => { | |
783 | if let Primitive::Int(Integer::I8, _) = s.value { | |
784 | cx.type_vector(cx.type_i8(), 8) | |
785 | } else { | |
786 | layout.llvm_type(cx) | |
787 | } | |
788 | } | |
789 | (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) => { | |
790 | let elem_ty = llvm_asm_scalar_type(cx, s); | |
791 | let count = 16 / layout.size.bytes(); | |
792 | cx.type_vector(elem_ty, count) | |
793 | } | |
794 | ( | |
795 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), | |
796 | Abi::Vector { element, count }, | |
797 | ) if layout.size.bytes() == 8 => { | |
798 | let elem_ty = llvm_asm_scalar_type(cx, element); | |
799 | cx.type_vector(elem_ty, count * 2) | |
800 | } | |
801 | (InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd), Abi::Scalar(s)) | |
802 | if s.value == Primitive::F64 => | |
803 | { | |
804 | cx.type_i64() | |
805 | } | |
806 | ( | |
807 | InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg), | |
808 | Abi::Vector { .. }, | |
809 | ) if layout.size.bytes() == 64 => cx.type_vector(cx.type_f64(), 8), | |
3dfed10e XL |
810 | ( |
811 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16), | |
812 | Abi::Scalar(s), | |
813 | ) => { | |
814 | if let Primitive::Int(Integer::I32, _) = s.value { | |
815 | cx.type_f32() | |
816 | } else { | |
817 | layout.llvm_type(cx) | |
818 | } | |
819 | } | |
f9f354fc XL |
820 | ( |
821 | InlineAsmRegClass::Arm( | |
3dfed10e | 822 | ArmInlineAsmRegClass::dreg |
f9f354fc | 823 | | ArmInlineAsmRegClass::dreg_low8 |
3dfed10e | 824 | | ArmInlineAsmRegClass::dreg_low16, |
f9f354fc XL |
825 | ), |
826 | Abi::Scalar(s), | |
827 | ) => { | |
3dfed10e XL |
828 | if let Primitive::Int(Integer::I64, _) = s.value { |
829 | cx.type_f64() | |
f9f354fc XL |
830 | } else { |
831 | layout.llvm_type(cx) | |
832 | } | |
833 | } | |
834 | _ => layout.llvm_type(cx), | |
835 | } | |
836 | } |