]>
Commit | Line | Data |
---|---|---|
5e7ed085 | 1 | use crate::attributes; |
dfeec247 | 2 | use crate::builder::Builder; |
a2a8927a | 3 | use crate::common::Funclet; |
9fa01778 | 4 | use crate::context::CodegenCx; |
dfeec247 | 5 | use crate::llvm; |
f9f354fc | 6 | use crate::type_::Type; |
9fa01778 | 7 | use crate::type_of::LayoutLlvmExt; |
9fa01778 | 8 | use crate::value::Value; |
54a0048b | 9 | |
3dfed10e | 10 | use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece}; |
a1dfa0c6 | 11 | use rustc_codegen_ssa::mir::operand::OperandValue; |
dfeec247 | 12 | use rustc_codegen_ssa::traits::*; |
f9f354fc | 13 | use rustc_data_structures::fx::FxHashMap; |
f9f354fc | 14 | use rustc_middle::ty::layout::TyAndLayout; |
3c0e092e | 15 | use rustc_middle::{bug, span_bug, ty::Instance}; |
5099ac24 | 16 | use rustc_span::{Pos, Span}; |
f9f354fc XL |
17 | use rustc_target::abi::*; |
18 | use rustc_target::asm::*; | |
32a655c1 | 19 | |
dfeec247 | 20 | use libc::{c_char, c_uint}; |
5e7ed085 | 21 | use smallvec::SmallVec; |
a1dfa0c6 | 22 | |
a2a8927a | 23 | impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { |
f9f354fc XL |
24 | fn codegen_inline_asm( |
25 | &mut self, | |
26 | template: &[InlineAsmTemplatePiece], | |
27 | operands: &[InlineAsmOperandRef<'tcx, Self>], | |
28 | options: InlineAsmOptions, | |
29 | line_spans: &[Span], | |
3c0e092e | 30 | instance: Instance<'_>, |
a2a8927a | 31 | dest_catch_funclet: Option<(Self::BasicBlock, Self::BasicBlock, Option<&Self::Funclet>)>, |
f9f354fc XL |
32 | ) { |
33 | let asm_arch = self.tcx.sess.asm_arch.unwrap(); | |
54a0048b | 34 | |
f9f354fc XL |
35 | // Collect the types of output operands |
36 | let mut constraints = vec![]; | |
cdc7bbd5 | 37 | let mut clobbers = vec![]; |
f9f354fc XL |
38 | let mut output_types = vec![]; |
39 | let mut op_idx = FxHashMap::default(); | |
136023e0 | 40 | let mut clobbered_x87 = false; |
f9f354fc XL |
41 | for (idx, op) in operands.iter().enumerate() { |
42 | match *op { | |
43 | InlineAsmOperandRef::Out { reg, late, place } => { | |
cdc7bbd5 XL |
44 | let is_target_supported = |reg_class: InlineAsmRegClass| { |
45 | for &(_, feature) in reg_class.supported_types(asm_arch) { | |
46 | if let Some(feature) = feature { | |
3c0e092e | 47 | let codegen_fn_attrs = self.tcx.codegen_fn_attrs(instance.def_id()); |
5099ac24 FG |
48 | if self.tcx.sess.target_features.contains(&feature) |
49 | || codegen_fn_attrs.target_features.contains(&feature) | |
cdc7bbd5 XL |
50 | { |
51 | return true; | |
52 | } | |
53 | } else { | |
54 | // Register class is unconditionally supported | |
55 | return true; | |
56 | } | |
57 | } | |
58 | false | |
59 | }; | |
60 | ||
3dfed10e XL |
61 | let mut layout = None; |
62 | let ty = if let Some(ref place) = place { | |
63 | layout = Some(&place.layout); | |
f9f354fc | 64 | llvm_fixup_output_type(self.cx, reg.reg_class(), &place.layout) |
136023e0 XL |
65 | } else if matches!( |
66 | reg.reg_class(), | |
67 | InlineAsmRegClass::X86( | |
68 | X86InlineAsmRegClass::mmx_reg | X86InlineAsmRegClass::x87_reg | |
69 | ) | |
70 | ) { | |
71 | // Special handling for x87/mmx registers: we always | |
72 | // clobber the whole set if one register is marked as | |
73 | // clobbered. This is due to the way LLVM handles the | |
74 | // FP stack in inline assembly. | |
75 | if !clobbered_x87 { | |
76 | clobbered_x87 = true; | |
77 | clobbers.push("~{st}".to_string()); | |
78 | for i in 1..=7 { | |
79 | clobbers.push(format!("~{{st({})}}", i)); | |
80 | } | |
81 | } | |
82 | continue; | |
83 | } else if !is_target_supported(reg.reg_class()) | |
84 | || reg.reg_class().is_clobber_only(asm_arch) | |
85 | { | |
cdc7bbd5 XL |
86 | // We turn discarded outputs into clobber constraints |
87 | // if the target feature needed by the register class is | |
88 | // disabled. This is necessary otherwise LLVM will try | |
89 | // to actually allocate a register for the dummy output. | |
90 | assert!(matches!(reg, InlineAsmRegOrRegClass::Reg(_))); | |
91 | clobbers.push(format!("~{}", reg_to_llvm(reg, None))); | |
92 | continue; | |
f9f354fc XL |
93 | } else { |
94 | // If the output is discarded, we don't really care what | |
95 | // type is used. We're just using this to tell LLVM to | |
96 | // reserve the register. | |
97 | dummy_output_type(self.cx, reg.reg_class()) | |
98 | }; | |
99 | output_types.push(ty); | |
100 | op_idx.insert(idx, constraints.len()); | |
101 | let prefix = if late { "=" } else { "=&" }; | |
3dfed10e | 102 | constraints.push(format!("{}{}", prefix, reg_to_llvm(reg, layout))); |
f9f354fc XL |
103 | } |
104 | InlineAsmOperandRef::InOut { reg, late, in_value, out_place } => { | |
3dfed10e XL |
105 | let layout = if let Some(ref out_place) = out_place { |
106 | &out_place.layout | |
f9f354fc XL |
107 | } else { |
108 | // LLVM required tied operands to have the same type, | |
109 | // so we just use the type of the input. | |
3dfed10e | 110 | &in_value.layout |
f9f354fc | 111 | }; |
3dfed10e | 112 | let ty = llvm_fixup_output_type(self.cx, reg.reg_class(), layout); |
f9f354fc XL |
113 | output_types.push(ty); |
114 | op_idx.insert(idx, constraints.len()); | |
115 | let prefix = if late { "=" } else { "=&" }; | |
3dfed10e | 116 | constraints.push(format!("{}{}", prefix, reg_to_llvm(reg, Some(layout)))); |
f9f354fc XL |
117 | } |
118 | _ => {} | |
119 | } | |
a1dfa0c6 | 120 | } |
0bf4aa26 | 121 | |
f9f354fc XL |
122 | // Collect input operands |
123 | let mut inputs = vec![]; | |
124 | for (idx, op) in operands.iter().enumerate() { | |
125 | match *op { | |
126 | InlineAsmOperandRef::In { reg, value } => { | |
3dfed10e | 127 | let llval = |
f9f354fc | 128 | llvm_fixup_input(self, value.immediate(), reg.reg_class(), &value.layout); |
3dfed10e | 129 | inputs.push(llval); |
f9f354fc | 130 | op_idx.insert(idx, constraints.len()); |
3dfed10e | 131 | constraints.push(reg_to_llvm(reg, Some(&value.layout))); |
f9f354fc XL |
132 | } |
133 | InlineAsmOperandRef::InOut { reg, late: _, in_value, out_place: _ } => { | |
134 | let value = llvm_fixup_input( | |
135 | self, | |
136 | in_value.immediate(), | |
137 | reg.reg_class(), | |
138 | &in_value.layout, | |
139 | ); | |
140 | inputs.push(value); | |
141 | constraints.push(format!("{}", op_idx[&idx])); | |
142 | } | |
143 | InlineAsmOperandRef::SymFn { instance } => { | |
144 | inputs.push(self.cx.get_fn(instance)); | |
145 | op_idx.insert(idx, constraints.len()); | |
146 | constraints.push("s".to_string()); | |
147 | } | |
148 | InlineAsmOperandRef::SymStatic { def_id } => { | |
149 | inputs.push(self.cx.get_static(def_id)); | |
150 | op_idx.insert(idx, constraints.len()); | |
151 | constraints.push("s".to_string()); | |
152 | } | |
153 | _ => {} | |
154 | } | |
155 | } | |
156 | ||
157 | // Build the template string | |
158 | let mut template_str = String::new(); | |
159 | for piece in template { | |
160 | match *piece { | |
161 | InlineAsmTemplatePiece::String(ref s) => { | |
162 | if s.contains('$') { | |
163 | for c in s.chars() { | |
164 | if c == '$' { | |
165 | template_str.push_str("$$"); | |
166 | } else { | |
167 | template_str.push(c); | |
168 | } | |
169 | } | |
170 | } else { | |
171 | template_str.push_str(s) | |
172 | } | |
173 | } | |
174 | InlineAsmTemplatePiece::Placeholder { operand_idx, modifier, span: _ } => { | |
175 | match operands[operand_idx] { | |
176 | InlineAsmOperandRef::In { reg, .. } | |
177 | | InlineAsmOperandRef::Out { reg, .. } | |
178 | | InlineAsmOperandRef::InOut { reg, .. } => { | |
179 | let modifier = modifier_to_llvm(asm_arch, reg.reg_class(), modifier); | |
180 | if let Some(modifier) = modifier { | |
181 | template_str.push_str(&format!( | |
182 | "${{{}:{}}}", | |
183 | op_idx[&operand_idx], modifier | |
184 | )); | |
185 | } else { | |
186 | template_str.push_str(&format!("${{{}}}", op_idx[&operand_idx])); | |
187 | } | |
188 | } | |
189 | InlineAsmOperandRef::Const { ref string } => { | |
190 | // Const operands get injected directly into the template | |
191 | template_str.push_str(string); | |
192 | } | |
193 | InlineAsmOperandRef::SymFn { .. } | |
194 | | InlineAsmOperandRef::SymStatic { .. } => { | |
195 | // Only emit the raw symbol name | |
196 | template_str.push_str(&format!("${{{}:c}}", op_idx[&operand_idx])); | |
197 | } | |
198 | } | |
199 | } | |
200 | } | |
201 | } | |
202 | ||
cdc7bbd5 | 203 | constraints.append(&mut clobbers); |
f9f354fc XL |
204 | if !options.contains(InlineAsmOptions::PRESERVES_FLAGS) { |
205 | match asm_arch { | |
206 | InlineAsmArch::AArch64 | InlineAsmArch::Arm => { | |
207 | constraints.push("~{cc}".to_string()); | |
208 | } | |
209 | InlineAsmArch::X86 | InlineAsmArch::X86_64 => { | |
210 | constraints.extend_from_slice(&[ | |
211 | "~{dirflag}".to_string(), | |
212 | "~{fpsr}".to_string(), | |
213 | "~{flags}".to_string(), | |
214 | ]); | |
215 | } | |
94222f64 XL |
216 | InlineAsmArch::RiscV32 | InlineAsmArch::RiscV64 => { |
217 | constraints.extend_from_slice(&[ | |
218 | "~{vtype}".to_string(), | |
219 | "~{vl}".to_string(), | |
220 | "~{vxsat}".to_string(), | |
221 | "~{vxrm}".to_string(), | |
222 | ]); | |
223 | } | |
a2a8927a XL |
224 | InlineAsmArch::Avr => { |
225 | constraints.push("~{sreg}".to_string()); | |
226 | } | |
f9f354fc | 227 | InlineAsmArch::Nvptx64 => {} |
17df50a5 | 228 | InlineAsmArch::PowerPC | InlineAsmArch::PowerPC64 => {} |
f035d41b | 229 | InlineAsmArch::Hexagon => {} |
29967ef6 | 230 | InlineAsmArch::Mips | InlineAsmArch::Mips64 => {} |
94222f64 | 231 | InlineAsmArch::S390x => {} |
29967ef6 | 232 | InlineAsmArch::SpirV => {} |
3c0e092e | 233 | InlineAsmArch::Wasm32 | InlineAsmArch::Wasm64 => {} |
17df50a5 | 234 | InlineAsmArch::Bpf => {} |
5099ac24 FG |
235 | InlineAsmArch::Msp430 => { |
236 | constraints.push("~{sr}".to_string()); | |
237 | } | |
f9f354fc XL |
238 | } |
239 | } | |
240 | if !options.contains(InlineAsmOptions::NOMEM) { | |
241 | // This is actually ignored by LLVM, but it's probably best to keep | |
242 | // it just in case. LLVM instead uses the ReadOnly/ReadNone | |
243 | // attributes on the call instruction to optimize. | |
244 | constraints.push("~{memory}".to_string()); | |
245 | } | |
246 | let volatile = !options.contains(InlineAsmOptions::PURE); | |
247 | let alignstack = !options.contains(InlineAsmOptions::NOSTACK); | |
248 | let output_type = match &output_types[..] { | |
249 | [] => self.type_void(), | |
250 | [ty] => ty, | |
c295e0f8 | 251 | tys => self.type_struct(tys, false), |
f9f354fc XL |
252 | }; |
253 | let dialect = match asm_arch { | |
254 | InlineAsmArch::X86 | InlineAsmArch::X86_64 | |
255 | if !options.contains(InlineAsmOptions::ATT_SYNTAX) => | |
256 | { | |
5099ac24 | 257 | llvm::AsmDialect::Intel |
f9f354fc | 258 | } |
5099ac24 | 259 | _ => llvm::AsmDialect::Att, |
f9f354fc XL |
260 | }; |
261 | let result = inline_asm_call( | |
262 | self, | |
263 | &template_str, | |
264 | &constraints.join(","), | |
265 | &inputs, | |
266 | output_type, | |
267 | volatile, | |
268 | alignstack, | |
269 | dialect, | |
270 | line_spans, | |
a2a8927a XL |
271 | options.contains(InlineAsmOptions::MAY_UNWIND), |
272 | dest_catch_funclet, | |
f9f354fc XL |
273 | ) |
274 | .unwrap_or_else(|| span_bug!(line_spans[0], "LLVM asm constraint validation failed")); | |
275 | ||
5e7ed085 | 276 | let mut attrs = SmallVec::<[_; 2]>::new(); |
f9f354fc XL |
277 | if options.contains(InlineAsmOptions::PURE) { |
278 | if options.contains(InlineAsmOptions::NOMEM) { | |
5e7ed085 | 279 | attrs.push(llvm::AttributeKind::ReadNone.create_attr(self.cx.llcx)); |
f9f354fc | 280 | } else if options.contains(InlineAsmOptions::READONLY) { |
5e7ed085 | 281 | attrs.push(llvm::AttributeKind::ReadOnly.create_attr(self.cx.llcx)); |
f9f354fc | 282 | } |
5e7ed085 | 283 | attrs.push(llvm::AttributeKind::WillReturn.create_attr(self.cx.llcx)); |
29967ef6 | 284 | } else if options.contains(InlineAsmOptions::NOMEM) { |
5e7ed085 | 285 | attrs.push(llvm::AttributeKind::InaccessibleMemOnly.create_attr(self.cx.llcx)); |
f9f354fc | 286 | } else { |
29967ef6 | 287 | // LLVM doesn't have an attribute to represent ReadOnly + SideEffect |
f9f354fc | 288 | } |
5e7ed085 | 289 | attributes::apply_to_callsite(result, llvm::AttributePlace::Function, &{ attrs }); |
f9f354fc | 290 | |
04454e1e FG |
291 | // Switch to the 'normal' basic block if we did an `invoke` instead of a `call` |
292 | if let Some((dest, _, _)) = dest_catch_funclet { | |
293 | self.switch_to_block(dest); | |
294 | } | |
295 | ||
f9f354fc XL |
296 | // Write results to outputs |
297 | for (idx, op) in operands.iter().enumerate() { | |
298 | if let InlineAsmOperandRef::Out { reg, place: Some(place), .. } | |
299 | | InlineAsmOperandRef::InOut { reg, out_place: Some(place), .. } = *op | |
300 | { | |
301 | let value = if output_types.len() == 1 { | |
302 | result | |
303 | } else { | |
304 | self.extract_value(result, op_idx[&idx] as u64) | |
305 | }; | |
306 | let value = llvm_fixup_output(self, value, reg.reg_class(), &place.layout); | |
307 | OperandValue::Immediate(value).store(self, place); | |
308 | } | |
309 | } | |
a1dfa0c6 | 310 | } |
54a0048b | 311 | } |
cc61c64b | 312 | |
04454e1e | 313 | impl<'tcx> AsmMethods<'tcx> for CodegenCx<'_, 'tcx> { |
17df50a5 XL |
314 | fn codegen_global_asm( |
315 | &self, | |
316 | template: &[InlineAsmTemplatePiece], | |
04454e1e | 317 | operands: &[GlobalAsmOperandRef<'tcx>], |
17df50a5 XL |
318 | options: InlineAsmOptions, |
319 | _line_spans: &[Span], | |
320 | ) { | |
321 | let asm_arch = self.tcx.sess.asm_arch.unwrap(); | |
322 | ||
323 | // Default to Intel syntax on x86 | |
324 | let intel_syntax = matches!(asm_arch, InlineAsmArch::X86 | InlineAsmArch::X86_64) | |
325 | && !options.contains(InlineAsmOptions::ATT_SYNTAX); | |
326 | ||
327 | // Build the template string | |
328 | let mut template_str = String::new(); | |
329 | if intel_syntax { | |
330 | template_str.push_str(".intel_syntax\n"); | |
331 | } | |
332 | for piece in template { | |
333 | match *piece { | |
334 | InlineAsmTemplatePiece::String(ref s) => template_str.push_str(s), | |
335 | InlineAsmTemplatePiece::Placeholder { operand_idx, modifier: _, span: _ } => { | |
336 | match operands[operand_idx] { | |
337 | GlobalAsmOperandRef::Const { ref string } => { | |
338 | // Const operands get injected directly into the | |
339 | // template. Note that we don't need to escape $ | |
340 | // here unlike normal inline assembly. | |
341 | template_str.push_str(string); | |
342 | } | |
04454e1e FG |
343 | GlobalAsmOperandRef::SymFn { instance } => { |
344 | let llval = self.get_fn(instance); | |
345 | self.add_compiler_used_global(llval); | |
346 | let symbol = llvm::build_string(|s| unsafe { | |
347 | llvm::LLVMRustGetMangledName(llval, s); | |
348 | }) | |
349 | .expect("symbol is not valid UTF-8"); | |
350 | template_str.push_str(&symbol); | |
351 | } | |
352 | GlobalAsmOperandRef::SymStatic { def_id } => { | |
353 | let llval = self | |
354 | .renamed_statics | |
355 | .borrow() | |
356 | .get(&def_id) | |
357 | .copied() | |
358 | .unwrap_or_else(|| self.get_static(def_id)); | |
359 | self.add_compiler_used_global(llval); | |
360 | let symbol = llvm::build_string(|s| unsafe { | |
361 | llvm::LLVMRustGetMangledName(llval, s); | |
362 | }) | |
363 | .expect("symbol is not valid UTF-8"); | |
364 | template_str.push_str(&symbol); | |
365 | } | |
17df50a5 XL |
366 | } |
367 | } | |
368 | } | |
369 | } | |
370 | if intel_syntax { | |
371 | template_str.push_str("\n.att_syntax\n"); | |
372 | } | |
373 | ||
a1dfa0c6 | 374 | unsafe { |
17df50a5 XL |
375 | llvm::LLVMRustAppendModuleInlineAsm( |
376 | self.llmod, | |
377 | template_str.as_ptr().cast(), | |
378 | template_str.len(), | |
379 | ); | |
a1dfa0c6 | 380 | } |
cc61c64b XL |
381 | } |
382 | } | |
532ac7d7 | 383 | |
a2a8927a XL |
384 | pub(crate) fn inline_asm_call<'ll>( |
385 | bx: &mut Builder<'_, 'll, '_>, | |
ba9703b0 XL |
386 | asm: &str, |
387 | cons: &str, | |
532ac7d7 XL |
388 | inputs: &[&'ll Value], |
389 | output: &'ll llvm::Type, | |
390 | volatile: bool, | |
391 | alignstack: bool, | |
5099ac24 | 392 | dia: llvm::AsmDialect, |
f9f354fc | 393 | line_spans: &[Span], |
a2a8927a XL |
394 | unwind: bool, |
395 | dest_catch_funclet: Option<( | |
396 | &'ll llvm::BasicBlock, | |
397 | &'ll llvm::BasicBlock, | |
398 | Option<&Funclet<'ll>>, | |
399 | )>, | |
532ac7d7 | 400 | ) -> Option<&'ll Value> { |
dfeec247 XL |
401 | let volatile = if volatile { llvm::True } else { llvm::False }; |
402 | let alignstack = if alignstack { llvm::True } else { llvm::False }; | |
a2a8927a | 403 | let can_throw = if unwind { llvm::True } else { llvm::False }; |
dfeec247 XL |
404 | |
405 | let argtys = inputs | |
406 | .iter() | |
407 | .map(|v| { | |
408 | debug!("Asm Input Type: {:?}", *v); | |
409 | bx.cx.val_ty(*v) | |
410 | }) | |
411 | .collect::<Vec<_>>(); | |
532ac7d7 XL |
412 | |
413 | debug!("Asm Output Type: {:?}", output); | |
a2a8927a | 414 | let fty = bx.cx.type_func(&argtys, output); |
532ac7d7 XL |
415 | unsafe { |
416 | // Ask LLVM to verify that the constraints are well-formed. | |
ba9703b0 | 417 | let constraints_ok = llvm::LLVMRustInlineAsmVerify(fty, cons.as_ptr().cast(), cons.len()); |
416331ca | 418 | debug!("constraint verification result: {:?}", constraints_ok); |
532ac7d7 XL |
419 | if constraints_ok { |
420 | let v = llvm::LLVMRustInlineAsm( | |
421 | fty, | |
ba9703b0 XL |
422 | asm.as_ptr().cast(), |
423 | asm.len(), | |
424 | cons.as_ptr().cast(), | |
425 | cons.len(), | |
532ac7d7 XL |
426 | volatile, |
427 | alignstack, | |
5099ac24 | 428 | dia, |
a2a8927a | 429 | can_throw, |
532ac7d7 | 430 | ); |
a2a8927a XL |
431 | |
432 | let call = if let Some((dest, catch, funclet)) = dest_catch_funclet { | |
433 | bx.invoke(fty, v, inputs, dest, catch, funclet) | |
434 | } else { | |
435 | bx.call(fty, v, inputs, None) | |
436 | }; | |
f9f354fc XL |
437 | |
438 | // Store mark in a metadata node so we can map LLVM errors | |
439 | // back to source locations. See #17552. | |
440 | let key = "srcloc"; | |
441 | let kind = llvm::LLVMGetMDKindIDInContext( | |
442 | bx.llcx, | |
443 | key.as_ptr() as *const c_char, | |
444 | key.len() as c_uint, | |
445 | ); | |
446 | ||
447 | // srcloc contains one integer for each line of assembly code. | |
448 | // Unfortunately this isn't enough to encode a full span so instead | |
449 | // we just encode the start position of each line. | |
450 | // FIXME: Figure out a way to pass the entire line spans. | |
451 | let mut srcloc = vec![]; | |
5099ac24 | 452 | if dia == llvm::AsmDialect::Intel && line_spans.len() > 1 { |
f9f354fc XL |
453 | // LLVM inserts an extra line to add the ".intel_syntax", so add |
454 | // a dummy srcloc entry for it. | |
455 | // | |
456 | // Don't do this if we only have 1 line span since that may be | |
457 | // due to the asm template string coming from a macro. LLVM will | |
458 | // default to the first srcloc for lines that don't have an | |
459 | // associated srcloc. | |
460 | srcloc.push(bx.const_i32(0)); | |
461 | } | |
462 | srcloc.extend(line_spans.iter().map(|span| bx.const_i32(span.lo().to_u32() as i32))); | |
463 | let md = llvm::LLVMMDNodeInContext(bx.llcx, srcloc.as_ptr(), srcloc.len() as u32); | |
464 | llvm::LLVMSetMetadata(call, kind, md); | |
465 | ||
466 | Some(call) | |
532ac7d7 XL |
467 | } else { |
468 | // LLVM has detected an issue with our constraints, bail out | |
469 | None | |
470 | } | |
471 | } | |
472 | } | |
f9f354fc | 473 | |
3dfed10e XL |
474 | /// If the register is an xmm/ymm/zmm register then return its index. |
475 | fn xmm_reg_index(reg: InlineAsmReg) -> Option<u32> { | |
476 | match reg { | |
477 | InlineAsmReg::X86(reg) | |
478 | if reg as u32 >= X86InlineAsmReg::xmm0 as u32 | |
479 | && reg as u32 <= X86InlineAsmReg::xmm15 as u32 => | |
480 | { | |
481 | Some(reg as u32 - X86InlineAsmReg::xmm0 as u32) | |
482 | } | |
483 | InlineAsmReg::X86(reg) | |
484 | if reg as u32 >= X86InlineAsmReg::ymm0 as u32 | |
485 | && reg as u32 <= X86InlineAsmReg::ymm15 as u32 => | |
486 | { | |
487 | Some(reg as u32 - X86InlineAsmReg::ymm0 as u32) | |
488 | } | |
489 | InlineAsmReg::X86(reg) | |
490 | if reg as u32 >= X86InlineAsmReg::zmm0 as u32 | |
491 | && reg as u32 <= X86InlineAsmReg::zmm31 as u32 => | |
492 | { | |
493 | Some(reg as u32 - X86InlineAsmReg::zmm0 as u32) | |
494 | } | |
495 | _ => None, | |
496 | } | |
497 | } | |
498 | ||
499 | /// If the register is an AArch64 vector register then return its index. | |
500 | fn a64_vreg_index(reg: InlineAsmReg) -> Option<u32> { | |
501 | match reg { | |
502 | InlineAsmReg::AArch64(reg) | |
503 | if reg as u32 >= AArch64InlineAsmReg::v0 as u32 | |
504 | && reg as u32 <= AArch64InlineAsmReg::v31 as u32 => | |
505 | { | |
506 | Some(reg as u32 - AArch64InlineAsmReg::v0 as u32) | |
507 | } | |
508 | _ => None, | |
509 | } | |
510 | } | |
511 | ||
f9f354fc | 512 | /// Converts a register class to an LLVM constraint code. |
a2a8927a | 513 | fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> String { |
f9f354fc | 514 | match reg { |
3dfed10e XL |
515 | // For vector registers LLVM wants the register name to match the type size. |
516 | InlineAsmRegOrRegClass::Reg(reg) => { | |
517 | if let Some(idx) = xmm_reg_index(reg) { | |
518 | let class = if let Some(layout) = layout { | |
519 | match layout.size.bytes() { | |
520 | 64 => 'z', | |
521 | 32 => 'y', | |
522 | _ => 'x', | |
523 | } | |
524 | } else { | |
525 | // We use f32 as the type for discarded outputs | |
526 | 'x' | |
527 | }; | |
528 | format!("{{{}mm{}}}", class, idx) | |
529 | } else if let Some(idx) = a64_vreg_index(reg) { | |
530 | let class = if let Some(layout) = layout { | |
531 | match layout.size.bytes() { | |
532 | 16 => 'q', | |
533 | 8 => 'd', | |
534 | 4 => 's', | |
535 | 2 => 'h', | |
536 | 1 => 'd', // We fixup i8 to i8x8 | |
537 | _ => unreachable!(), | |
538 | } | |
539 | } else { | |
540 | // We use i64x2 as the type for discarded outputs | |
541 | 'q' | |
542 | }; | |
543 | format!("{{{}{}}}", class, idx) | |
544 | } else if reg == InlineAsmReg::AArch64(AArch64InlineAsmReg::x30) { | |
545 | // LLVM doesn't recognize x30 | |
1b1a35ee | 546 | "{lr}".to_string() |
6a06907d XL |
547 | } else if reg == InlineAsmReg::Arm(ArmInlineAsmReg::r14) { |
548 | // LLVM doesn't recognize r14 | |
549 | "{lr}".to_string() | |
3dfed10e XL |
550 | } else { |
551 | format!("{{{}}}", reg.name()) | |
552 | } | |
553 | } | |
f9f354fc XL |
554 | InlineAsmRegOrRegClass::RegClass(reg) => match reg { |
555 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => "r", | |
556 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) => "w", | |
557 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => "x", | |
136023e0 XL |
558 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => { |
559 | unreachable!("clobber-only") | |
560 | } | |
f9f354fc | 561 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) => "r", |
f9f354fc XL |
562 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg) |
563 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16) | |
564 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8) => "t", | |
565 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16) | |
566 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8) | |
567 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4) => "x", | |
568 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg) | |
569 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg) => "w", | |
f035d41b | 570 | InlineAsmRegClass::Hexagon(HexagonInlineAsmRegClass::reg) => "r", |
1b1a35ee XL |
571 | InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg) => "r", |
572 | InlineAsmRegClass::Mips(MipsInlineAsmRegClass::freg) => "f", | |
f9f354fc XL |
573 | InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg16) => "h", |
574 | InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg32) => "r", | |
575 | InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg64) => "l", | |
17df50a5 XL |
576 | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg) => "r", |
577 | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => "b", | |
578 | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::freg) => "f", | |
94222f64 XL |
579 | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::cr) |
580 | | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::xer) => { | |
581 | unreachable!("clobber-only") | |
582 | } | |
f9f354fc XL |
583 | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) => "r", |
584 | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => "f", | |
136023e0 XL |
585 | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::vreg) => { |
586 | unreachable!("clobber-only") | |
587 | } | |
f9f354fc XL |
588 | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg) => "r", |
589 | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => "Q", | |
590 | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_byte) => "q", | |
591 | InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg) | |
592 | | InlineAsmRegClass::X86(X86InlineAsmRegClass::ymm_reg) => "x", | |
593 | InlineAsmRegClass::X86(X86InlineAsmRegClass::zmm_reg) => "v", | |
594 | InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => "^Yk", | |
136023e0 | 595 | InlineAsmRegClass::X86( |
04454e1e FG |
596 | X86InlineAsmRegClass::x87_reg |
597 | | X86InlineAsmRegClass::mmx_reg | |
923072b8 FG |
598 | | X86InlineAsmRegClass::kreg0 |
599 | | X86InlineAsmRegClass::tmm_reg, | |
136023e0 | 600 | ) => unreachable!("clobber-only"), |
fc512014 | 601 | InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => "r", |
17df50a5 XL |
602 | InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::reg) => "r", |
603 | InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::wreg) => "w", | |
a2a8927a XL |
604 | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg) => "r", |
605 | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_upper) => "d", | |
606 | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_pair) => "r", | |
607 | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_iw) => "w", | |
608 | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_ptr) => "e", | |
94222f64 XL |
609 | InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg) => "r", |
610 | InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => "f", | |
5099ac24 | 611 | InlineAsmRegClass::Msp430(Msp430InlineAsmRegClass::reg) => "r", |
29967ef6 XL |
612 | InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { |
613 | bug!("LLVM backend does not support SPIR-V") | |
614 | } | |
6a06907d | 615 | InlineAsmRegClass::Err => unreachable!(), |
f9f354fc XL |
616 | } |
617 | .to_string(), | |
618 | } | |
619 | } | |
620 | ||
621 | /// Converts a modifier into LLVM's equivalent modifier. | |
622 | fn modifier_to_llvm( | |
623 | arch: InlineAsmArch, | |
624 | reg: InlineAsmRegClass, | |
625 | modifier: Option<char>, | |
626 | ) -> Option<char> { | |
627 | match reg { | |
628 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => modifier, | |
629 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) | |
630 | | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => { | |
631 | if modifier == Some('v') { None } else { modifier } | |
632 | } | |
136023e0 XL |
633 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => { |
634 | unreachable!("clobber-only") | |
635 | } | |
a2a8927a | 636 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) => None, |
f9f354fc XL |
637 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg) |
638 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16) => None, | |
639 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg) | |
640 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16) | |
641 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8) => Some('P'), | |
642 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg) | |
643 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8) | |
644 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4) => { | |
645 | if modifier.is_none() { | |
646 | Some('q') | |
647 | } else { | |
648 | modifier | |
649 | } | |
650 | } | |
f035d41b | 651 | InlineAsmRegClass::Hexagon(_) => None, |
1b1a35ee | 652 | InlineAsmRegClass::Mips(_) => None, |
f9f354fc | 653 | InlineAsmRegClass::Nvptx(_) => None, |
17df50a5 | 654 | InlineAsmRegClass::PowerPC(_) => None, |
f9f354fc XL |
655 | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) |
656 | | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => None, | |
136023e0 XL |
657 | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::vreg) => { |
658 | unreachable!("clobber-only") | |
659 | } | |
f9f354fc XL |
660 | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg) |
661 | | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => match modifier { | |
662 | None if arch == InlineAsmArch::X86_64 => Some('q'), | |
663 | None => Some('k'), | |
664 | Some('l') => Some('b'), | |
665 | Some('h') => Some('h'), | |
666 | Some('x') => Some('w'), | |
667 | Some('e') => Some('k'), | |
668 | Some('r') => Some('q'), | |
669 | _ => unreachable!(), | |
670 | }, | |
671 | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_byte) => None, | |
672 | InlineAsmRegClass::X86(reg @ X86InlineAsmRegClass::xmm_reg) | |
673 | | InlineAsmRegClass::X86(reg @ X86InlineAsmRegClass::ymm_reg) | |
674 | | InlineAsmRegClass::X86(reg @ X86InlineAsmRegClass::zmm_reg) => match (reg, modifier) { | |
675 | (X86InlineAsmRegClass::xmm_reg, None) => Some('x'), | |
676 | (X86InlineAsmRegClass::ymm_reg, None) => Some('t'), | |
677 | (X86InlineAsmRegClass::zmm_reg, None) => Some('g'), | |
678 | (_, Some('x')) => Some('x'), | |
679 | (_, Some('y')) => Some('t'), | |
680 | (_, Some('z')) => Some('g'), | |
681 | _ => unreachable!(), | |
682 | }, | |
683 | InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => None, | |
04454e1e FG |
684 | InlineAsmRegClass::X86( |
685 | X86InlineAsmRegClass::x87_reg | |
686 | | X86InlineAsmRegClass::mmx_reg | |
923072b8 FG |
687 | | X86InlineAsmRegClass::kreg0 |
688 | | X86InlineAsmRegClass::tmm_reg, | |
04454e1e | 689 | ) => { |
136023e0 XL |
690 | unreachable!("clobber-only") |
691 | } | |
fc512014 | 692 | InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => None, |
17df50a5 | 693 | InlineAsmRegClass::Bpf(_) => None, |
a2a8927a XL |
694 | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_pair) |
695 | | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_iw) | |
696 | | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_ptr) => match modifier { | |
697 | Some('h') => Some('B'), | |
698 | Some('l') => Some('A'), | |
699 | _ => None, | |
700 | }, | |
701 | InlineAsmRegClass::Avr(_) => None, | |
94222f64 | 702 | InlineAsmRegClass::S390x(_) => None, |
5099ac24 | 703 | InlineAsmRegClass::Msp430(_) => None, |
29967ef6 XL |
704 | InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { |
705 | bug!("LLVM backend does not support SPIR-V") | |
706 | } | |
6a06907d | 707 | InlineAsmRegClass::Err => unreachable!(), |
f9f354fc XL |
708 | } |
709 | } | |
710 | ||
711 | /// Type to use for outputs that are discarded. It doesn't really matter what | |
712 | /// the type is, as long as it is valid for the constraint code. | |
a2a8927a | 713 | fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &'ll Type { |
f9f354fc XL |
714 | match reg { |
715 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => cx.type_i32(), | |
716 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) | |
717 | | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => { | |
718 | cx.type_vector(cx.type_i64(), 2) | |
719 | } | |
136023e0 XL |
720 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => { |
721 | unreachable!("clobber-only") | |
722 | } | |
a2a8927a | 723 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) => cx.type_i32(), |
f9f354fc XL |
724 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg) |
725 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16) => cx.type_f32(), | |
726 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg) | |
727 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16) | |
728 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8) => cx.type_f64(), | |
729 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg) | |
730 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8) | |
731 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4) => { | |
732 | cx.type_vector(cx.type_i64(), 2) | |
733 | } | |
f035d41b | 734 | InlineAsmRegClass::Hexagon(HexagonInlineAsmRegClass::reg) => cx.type_i32(), |
1b1a35ee XL |
735 | InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg) => cx.type_i32(), |
736 | InlineAsmRegClass::Mips(MipsInlineAsmRegClass::freg) => cx.type_f32(), | |
f9f354fc XL |
737 | InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg16) => cx.type_i16(), |
738 | InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg32) => cx.type_i32(), | |
739 | InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg64) => cx.type_i64(), | |
17df50a5 XL |
740 | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg) => cx.type_i32(), |
741 | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => cx.type_i32(), | |
742 | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::freg) => cx.type_f64(), | |
94222f64 XL |
743 | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::cr) |
744 | | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::xer) => { | |
745 | unreachable!("clobber-only") | |
746 | } | |
f9f354fc XL |
747 | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) => cx.type_i32(), |
748 | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => cx.type_f32(), | |
136023e0 XL |
749 | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::vreg) => { |
750 | unreachable!("clobber-only") | |
751 | } | |
f9f354fc XL |
752 | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg) |
753 | | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => cx.type_i32(), | |
754 | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_byte) => cx.type_i8(), | |
755 | InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg) | |
756 | | InlineAsmRegClass::X86(X86InlineAsmRegClass::ymm_reg) | |
757 | | InlineAsmRegClass::X86(X86InlineAsmRegClass::zmm_reg) => cx.type_f32(), | |
758 | InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => cx.type_i16(), | |
04454e1e FG |
759 | InlineAsmRegClass::X86( |
760 | X86InlineAsmRegClass::x87_reg | |
761 | | X86InlineAsmRegClass::mmx_reg | |
923072b8 FG |
762 | | X86InlineAsmRegClass::kreg0 |
763 | | X86InlineAsmRegClass::tmm_reg, | |
04454e1e | 764 | ) => { |
136023e0 XL |
765 | unreachable!("clobber-only") |
766 | } | |
fc512014 | 767 | InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => cx.type_i32(), |
17df50a5 XL |
768 | InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::reg) => cx.type_i64(), |
769 | InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::wreg) => cx.type_i32(), | |
a2a8927a XL |
770 | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg) => cx.type_i8(), |
771 | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_upper) => cx.type_i8(), | |
772 | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_pair) => cx.type_i16(), | |
773 | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_iw) => cx.type_i16(), | |
774 | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_ptr) => cx.type_i16(), | |
94222f64 XL |
775 | InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg) => cx.type_i32(), |
776 | InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => cx.type_f64(), | |
5099ac24 | 777 | InlineAsmRegClass::Msp430(Msp430InlineAsmRegClass::reg) => cx.type_i16(), |
29967ef6 XL |
778 | InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { |
779 | bug!("LLVM backend does not support SPIR-V") | |
780 | } | |
6a06907d | 781 | InlineAsmRegClass::Err => unreachable!(), |
f9f354fc XL |
782 | } |
783 | } | |
784 | ||
785 | /// Helper function to get the LLVM type for a Scalar. Pointers are returned as | |
786 | /// the equivalent integer type. | |
a2a8927a | 787 | fn llvm_asm_scalar_type<'ll>(cx: &CodegenCx<'ll, '_>, scalar: Scalar) -> &'ll Type { |
04454e1e | 788 | match scalar.primitive() { |
f9f354fc XL |
789 | Primitive::Int(Integer::I8, _) => cx.type_i8(), |
790 | Primitive::Int(Integer::I16, _) => cx.type_i16(), | |
791 | Primitive::Int(Integer::I32, _) => cx.type_i32(), | |
792 | Primitive::Int(Integer::I64, _) => cx.type_i64(), | |
793 | Primitive::F32 => cx.type_f32(), | |
794 | Primitive::F64 => cx.type_f64(), | |
795 | Primitive::Pointer => cx.type_isize(), | |
796 | _ => unreachable!(), | |
797 | } | |
798 | } | |
799 | ||
800 | /// Fix up an input value to work around LLVM bugs. | |
a2a8927a XL |
801 | fn llvm_fixup_input<'ll, 'tcx>( |
802 | bx: &mut Builder<'_, 'll, 'tcx>, | |
f9f354fc XL |
803 | mut value: &'ll Value, |
804 | reg: InlineAsmRegClass, | |
805 | layout: &TyAndLayout<'tcx>, | |
806 | ) -> &'ll Value { | |
c295e0f8 | 807 | match (reg, layout.abi) { |
f9f354fc | 808 | (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => { |
04454e1e | 809 | if let Primitive::Int(Integer::I8, _) = s.primitive() { |
f9f354fc XL |
810 | let vec_ty = bx.cx.type_vector(bx.cx.type_i8(), 8); |
811 | bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0)) | |
812 | } else { | |
813 | value | |
814 | } | |
815 | } | |
816 | (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) => { | |
817 | let elem_ty = llvm_asm_scalar_type(bx.cx, s); | |
818 | let count = 16 / layout.size.bytes(); | |
819 | let vec_ty = bx.cx.type_vector(elem_ty, count); | |
04454e1e | 820 | if let Primitive::Pointer = s.primitive() { |
f9f354fc XL |
821 | value = bx.ptrtoint(value, bx.cx.type_isize()); |
822 | } | |
823 | bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0)) | |
824 | } | |
825 | ( | |
826 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), | |
827 | Abi::Vector { element, count }, | |
828 | ) if layout.size.bytes() == 8 => { | |
829 | let elem_ty = llvm_asm_scalar_type(bx.cx, element); | |
c295e0f8 | 830 | let vec_ty = bx.cx.type_vector(elem_ty, count); |
f9f354fc XL |
831 | let indices: Vec<_> = (0..count * 2).map(|x| bx.const_i32(x as i32)).collect(); |
832 | bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices)) | |
833 | } | |
834 | (InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd), Abi::Scalar(s)) | |
04454e1e | 835 | if s.primitive() == Primitive::F64 => |
f9f354fc XL |
836 | { |
837 | bx.bitcast(value, bx.cx.type_i64()) | |
838 | } | |
839 | ( | |
840 | InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg), | |
841 | Abi::Vector { .. }, | |
842 | ) if layout.size.bytes() == 64 => bx.bitcast(value, bx.cx.type_vector(bx.cx.type_f64(), 8)), | |
3dfed10e XL |
843 | ( |
844 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16), | |
845 | Abi::Scalar(s), | |
846 | ) => { | |
04454e1e | 847 | if let Primitive::Int(Integer::I32, _) = s.primitive() { |
3dfed10e XL |
848 | bx.bitcast(value, bx.cx.type_f32()) |
849 | } else { | |
850 | value | |
851 | } | |
852 | } | |
f9f354fc XL |
853 | ( |
854 | InlineAsmRegClass::Arm( | |
3dfed10e | 855 | ArmInlineAsmRegClass::dreg |
f9f354fc | 856 | | ArmInlineAsmRegClass::dreg_low8 |
3dfed10e | 857 | | ArmInlineAsmRegClass::dreg_low16, |
f9f354fc XL |
858 | ), |
859 | Abi::Scalar(s), | |
860 | ) => { | |
04454e1e | 861 | if let Primitive::Int(Integer::I64, _) = s.primitive() { |
3dfed10e | 862 | bx.bitcast(value, bx.cx.type_f64()) |
f9f354fc XL |
863 | } else { |
864 | value | |
865 | } | |
866 | } | |
04454e1e FG |
867 | (InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg), Abi::Scalar(s)) => { |
868 | match s.primitive() { | |
869 | // MIPS only supports register-length arithmetics. | |
870 | Primitive::Int(Integer::I8 | Integer::I16, _) => bx.zext(value, bx.cx.type_i32()), | |
871 | Primitive::F32 => bx.bitcast(value, bx.cx.type_i32()), | |
872 | Primitive::F64 => bx.bitcast(value, bx.cx.type_i64()), | |
873 | _ => value, | |
874 | } | |
875 | } | |
f9f354fc XL |
876 | _ => value, |
877 | } | |
878 | } | |
879 | ||
880 | /// Fix up an output value to work around LLVM bugs. | |
a2a8927a XL |
881 | fn llvm_fixup_output<'ll, 'tcx>( |
882 | bx: &mut Builder<'_, 'll, 'tcx>, | |
f9f354fc XL |
883 | mut value: &'ll Value, |
884 | reg: InlineAsmRegClass, | |
885 | layout: &TyAndLayout<'tcx>, | |
886 | ) -> &'ll Value { | |
c295e0f8 | 887 | match (reg, layout.abi) { |
f9f354fc | 888 | (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => { |
04454e1e | 889 | if let Primitive::Int(Integer::I8, _) = s.primitive() { |
f9f354fc XL |
890 | bx.extract_element(value, bx.const_i32(0)) |
891 | } else { | |
892 | value | |
893 | } | |
894 | } | |
895 | (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) => { | |
896 | value = bx.extract_element(value, bx.const_i32(0)); | |
04454e1e | 897 | if let Primitive::Pointer = s.primitive() { |
f9f354fc XL |
898 | value = bx.inttoptr(value, layout.llvm_type(bx.cx)); |
899 | } | |
900 | value | |
901 | } | |
902 | ( | |
903 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), | |
904 | Abi::Vector { element, count }, | |
905 | ) if layout.size.bytes() == 8 => { | |
906 | let elem_ty = llvm_asm_scalar_type(bx.cx, element); | |
c295e0f8 XL |
907 | let vec_ty = bx.cx.type_vector(elem_ty, count * 2); |
908 | let indices: Vec<_> = (0..count).map(|x| bx.const_i32(x as i32)).collect(); | |
f9f354fc XL |
909 | bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices)) |
910 | } | |
911 | (InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd), Abi::Scalar(s)) | |
04454e1e | 912 | if s.primitive() == Primitive::F64 => |
f9f354fc XL |
913 | { |
914 | bx.bitcast(value, bx.cx.type_f64()) | |
915 | } | |
916 | ( | |
917 | InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg), | |
918 | Abi::Vector { .. }, | |
919 | ) if layout.size.bytes() == 64 => bx.bitcast(value, layout.llvm_type(bx.cx)), | |
3dfed10e XL |
920 | ( |
921 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16), | |
922 | Abi::Scalar(s), | |
923 | ) => { | |
04454e1e | 924 | if let Primitive::Int(Integer::I32, _) = s.primitive() { |
3dfed10e XL |
925 | bx.bitcast(value, bx.cx.type_i32()) |
926 | } else { | |
927 | value | |
928 | } | |
929 | } | |
f9f354fc XL |
930 | ( |
931 | InlineAsmRegClass::Arm( | |
3dfed10e | 932 | ArmInlineAsmRegClass::dreg |
f9f354fc | 933 | | ArmInlineAsmRegClass::dreg_low8 |
3dfed10e | 934 | | ArmInlineAsmRegClass::dreg_low16, |
f9f354fc XL |
935 | ), |
936 | Abi::Scalar(s), | |
937 | ) => { | |
04454e1e | 938 | if let Primitive::Int(Integer::I64, _) = s.primitive() { |
3dfed10e | 939 | bx.bitcast(value, bx.cx.type_i64()) |
f9f354fc XL |
940 | } else { |
941 | value | |
942 | } | |
943 | } | |
04454e1e FG |
944 | (InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg), Abi::Scalar(s)) => { |
945 | match s.primitive() { | |
946 | // MIPS only supports register-length arithmetics. | |
947 | Primitive::Int(Integer::I8, _) => bx.trunc(value, bx.cx.type_i8()), | |
948 | Primitive::Int(Integer::I16, _) => bx.trunc(value, bx.cx.type_i16()), | |
949 | Primitive::F32 => bx.bitcast(value, bx.cx.type_f32()), | |
950 | Primitive::F64 => bx.bitcast(value, bx.cx.type_f64()), | |
951 | _ => value, | |
952 | } | |
953 | } | |
f9f354fc XL |
954 | _ => value, |
955 | } | |
956 | } | |
957 | ||
958 | /// Output type to use for llvm_fixup_output. | |
a2a8927a | 959 | fn llvm_fixup_output_type<'ll, 'tcx>( |
f9f354fc XL |
960 | cx: &CodegenCx<'ll, 'tcx>, |
961 | reg: InlineAsmRegClass, | |
962 | layout: &TyAndLayout<'tcx>, | |
963 | ) -> &'ll Type { | |
c295e0f8 | 964 | match (reg, layout.abi) { |
f9f354fc | 965 | (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => { |
04454e1e | 966 | if let Primitive::Int(Integer::I8, _) = s.primitive() { |
f9f354fc XL |
967 | cx.type_vector(cx.type_i8(), 8) |
968 | } else { | |
969 | layout.llvm_type(cx) | |
970 | } | |
971 | } | |
972 | (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) => { | |
973 | let elem_ty = llvm_asm_scalar_type(cx, s); | |
974 | let count = 16 / layout.size.bytes(); | |
975 | cx.type_vector(elem_ty, count) | |
976 | } | |
977 | ( | |
978 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), | |
979 | Abi::Vector { element, count }, | |
980 | ) if layout.size.bytes() == 8 => { | |
981 | let elem_ty = llvm_asm_scalar_type(cx, element); | |
982 | cx.type_vector(elem_ty, count * 2) | |
983 | } | |
984 | (InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd), Abi::Scalar(s)) | |
04454e1e | 985 | if s.primitive() == Primitive::F64 => |
f9f354fc XL |
986 | { |
987 | cx.type_i64() | |
988 | } | |
989 | ( | |
990 | InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg), | |
991 | Abi::Vector { .. }, | |
992 | ) if layout.size.bytes() == 64 => cx.type_vector(cx.type_f64(), 8), | |
3dfed10e XL |
993 | ( |
994 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16), | |
995 | Abi::Scalar(s), | |
996 | ) => { | |
04454e1e | 997 | if let Primitive::Int(Integer::I32, _) = s.primitive() { |
3dfed10e XL |
998 | cx.type_f32() |
999 | } else { | |
1000 | layout.llvm_type(cx) | |
1001 | } | |
1002 | } | |
f9f354fc XL |
1003 | ( |
1004 | InlineAsmRegClass::Arm( | |
3dfed10e | 1005 | ArmInlineAsmRegClass::dreg |
f9f354fc | 1006 | | ArmInlineAsmRegClass::dreg_low8 |
3dfed10e | 1007 | | ArmInlineAsmRegClass::dreg_low16, |
f9f354fc XL |
1008 | ), |
1009 | Abi::Scalar(s), | |
1010 | ) => { | |
04454e1e | 1011 | if let Primitive::Int(Integer::I64, _) = s.primitive() { |
3dfed10e | 1012 | cx.type_f64() |
f9f354fc XL |
1013 | } else { |
1014 | layout.llvm_type(cx) | |
1015 | } | |
1016 | } | |
04454e1e FG |
1017 | (InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg), Abi::Scalar(s)) => { |
1018 | match s.primitive() { | |
1019 | // MIPS only supports register-length arithmetics. | |
1020 | Primitive::Int(Integer::I8 | Integer::I16, _) => cx.type_i32(), | |
1021 | Primitive::F32 => cx.type_i32(), | |
1022 | Primitive::F64 => cx.type_i64(), | |
1023 | _ => layout.llvm_type(cx), | |
1024 | } | |
1025 | } | |
f9f354fc XL |
1026 | _ => layout.llvm_type(cx), |
1027 | } | |
1028 | } |