]> git.proxmox.com Git - rustc.git/blame - compiler/rustc_codegen_llvm/src/asm.rs
projects / rustc.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
New upstream version 1.66.0+dfsg1
[rustc.git] / compiler / rustc_codegen_llvm / src / asm.rs
CommitLineData
5e7ed085 1use crate::attributes;
dfeec247 2use crate::builder::Builder;
a2a8927a 3use crate::common::Funclet;
9fa01778 4use crate::context::CodegenCx;
dfeec247 5use crate::llvm;
f9f354fc 6use crate::type_::Type;
9fa01778 7use crate::type_of::LayoutLlvmExt;
9fa01778 8use crate::value::Value;
54a0048b 9
3dfed10e 10use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
a1dfa0c6 11use rustc_codegen_ssa::mir::operand::OperandValue;
dfeec247 12use rustc_codegen_ssa::traits::*;
f9f354fc 13use rustc_data_structures::fx::FxHashMap;
f9f354fc 14use rustc_middle::ty::layout::TyAndLayout;
3c0e092e 15use rustc_middle::{bug, span_bug, ty::Instance};
5099ac24 16use rustc_span::{Pos, Span};
f9f354fc
XL		 17use rustc_target::abi::*;
18use rustc_target::asm::*;
32a655c1 19
dfeec247 20use libc::{c_char, c_uint};
5e7ed085 21use smallvec::SmallVec;
a1dfa0c6 22
a2a8927a 23impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
f9f354fc
XL		 24 fn codegen_inline_asm(
25 &mut self,
26 template: &[InlineAsmTemplatePiece],
27 operands: &[InlineAsmOperandRef<'tcx, Self>],
28 options: InlineAsmOptions,
29 line_spans: &[Span],
3c0e092e 30 instance: Instance<'_>,
a2a8927a 31 dest_catch_funclet: Option<(Self::BasicBlock, Self::BasicBlock, Option<&Self::Funclet>)>,
f9f354fc
XL		 32 ) {
33 let asm_arch = self.tcx.sess.asm_arch.unwrap();
54a0048b 34
f9f354fc
XL		 35 // Collect the types of output operands
36 let mut constraints = vec![];
cdc7bbd5 37 let mut clobbers = vec![];
f9f354fc
XL		 38 let mut output_types = vec![];
39 let mut op_idx = FxHashMap::default();
136023e0 40 let mut clobbered_x87 = false;
f9f354fc
XL		 41 for (idx, op) in operands.iter().enumerate() {
42 match *op {
43 InlineAsmOperandRef::Out { reg, late, place } => {
cdc7bbd5
XL		 44 let is_target_supported = |reg_class: InlineAsmRegClass| {
45 for &(_, feature) in reg_class.supported_types(asm_arch) {
46 if let Some(feature) = feature {
3c0e092e 47 let codegen_fn_attrs = self.tcx.codegen_fn_attrs(instance.def_id());
5099ac24
FG		 48 if self.tcx.sess.target_features.contains(&feature)
49 || codegen_fn_attrs.target_features.contains(&feature)
cdc7bbd5
XL		 50 {
51 return true;
52 }
53 } else {
54 // Register class is unconditionally supported
55 return true;
56 }
57 }
58 false
59 };
60
3dfed10e
XL		 61 let mut layout = None;
62 let ty = if let Some(ref place) = place {
63 layout = Some(&place.layout);
f9f354fc 64 llvm_fixup_output_type(self.cx, reg.reg_class(), &place.layout)
136023e0
XL		 65 } else if matches!(
66 reg.reg_class(),
67 InlineAsmRegClass::X86(
68 X86InlineAsmRegClass::mmx_reg | X86InlineAsmRegClass::x87_reg
69 )
70 ) {
71 // Special handling for x87/mmx registers: we always
72 // clobber the whole set if one register is marked as
73 // clobbered. This is due to the way LLVM handles the
74 // FP stack in inline assembly.
75 if !clobbered_x87 {
76 clobbered_x87 = true;
77 clobbers.push("~{st}".to_string());
78 for i in 1..=7 {
79 clobbers.push(format!("~{{st({})}}", i));
80 }
81 }
82 continue;
83 } else if !is_target_supported(reg.reg_class())
84 || reg.reg_class().is_clobber_only(asm_arch)
85 {
cdc7bbd5
XL		 86 // We turn discarded outputs into clobber constraints
87 // if the target feature needed by the register class is
88 // disabled. This is necessary otherwise LLVM will try
89 // to actually allocate a register for the dummy output.
90 assert!(matches!(reg, InlineAsmRegOrRegClass::Reg(_)));
91 clobbers.push(format!("~{}", reg_to_llvm(reg, None)));
92 continue;
f9f354fc
XL		 93 } else {
94 // If the output is discarded, we don't really care what
95 // type is used. We're just using this to tell LLVM to
96 // reserve the register.
97 dummy_output_type(self.cx, reg.reg_class())
98 };
99 output_types.push(ty);
100 op_idx.insert(idx, constraints.len());
101 let prefix = if late { "=" } else { "=&" };
3dfed10e 102 constraints.push(format!("{}{}", prefix, reg_to_llvm(reg, layout)));
f9f354fc
XL		 103 }
104 InlineAsmOperandRef::InOut { reg, late, in_value, out_place } => {
3dfed10e
XL		 105 let layout = if let Some(ref out_place) = out_place {
106 &out_place.layout
f9f354fc
XL		 107 } else {
108 // LLVM required tied operands to have the same type,
109 // so we just use the type of the input.
3dfed10e 110 &in_value.layout
f9f354fc 111 };
3dfed10e 112 let ty = llvm_fixup_output_type(self.cx, reg.reg_class(), layout);
f9f354fc
XL		 113 output_types.push(ty);
114 op_idx.insert(idx, constraints.len());
115 let prefix = if late { "=" } else { "=&" };
3dfed10e 116 constraints.push(format!("{}{}", prefix, reg_to_llvm(reg, Some(layout))));
f9f354fc
XL		 117 }
118 _ => {}
119 }
a1dfa0c6 120 }
0bf4aa26 121
f9f354fc
XL		 122 // Collect input operands
123 let mut inputs = vec![];
124 for (idx, op) in operands.iter().enumerate() {
125 match *op {
126 InlineAsmOperandRef::In { reg, value } => {
3dfed10e 127 let llval =
f9f354fc 128 llvm_fixup_input(self, value.immediate(), reg.reg_class(), &value.layout);
3dfed10e 129 inputs.push(llval);
f9f354fc 130 op_idx.insert(idx, constraints.len());
3dfed10e 131 constraints.push(reg_to_llvm(reg, Some(&value.layout)));
f9f354fc
XL		 132 }
133 InlineAsmOperandRef::InOut { reg, late: _, in_value, out_place: _ } => {
134 let value = llvm_fixup_input(
135 self,
136 in_value.immediate(),
137 reg.reg_class(),
138 &in_value.layout,
139 );
140 inputs.push(value);
141 constraints.push(format!("{}", op_idx[&idx]));
142 }
143 InlineAsmOperandRef::SymFn { instance } => {
144 inputs.push(self.cx.get_fn(instance));
145 op_idx.insert(idx, constraints.len());
146 constraints.push("s".to_string());
147 }
148 InlineAsmOperandRef::SymStatic { def_id } => {
149 inputs.push(self.cx.get_static(def_id));
150 op_idx.insert(idx, constraints.len());
151 constraints.push("s".to_string());
152 }
153 _ => {}
154 }
155 }
156
157 // Build the template string
158 let mut template_str = String::new();
159 for piece in template {
160 match *piece {
161 InlineAsmTemplatePiece::String(ref s) => {
162 if s.contains('$') {
163 for c in s.chars() {
164 if c == '$' {
165 template_str.push_str("$$");
166 } else {
167 template_str.push(c);
168 }
169 }
170 } else {
171 template_str.push_str(s)
172 }
173 }
174 InlineAsmTemplatePiece::Placeholder { operand_idx, modifier, span: _ } => {
175 match operands[operand_idx] {
176 InlineAsmOperandRef::In { reg, .. }
177 | InlineAsmOperandRef::Out { reg, .. }
178 | InlineAsmOperandRef::InOut { reg, .. } => {
179 let modifier = modifier_to_llvm(asm_arch, reg.reg_class(), modifier);
180 if let Some(modifier) = modifier {
181 template_str.push_str(&format!(
182 "${{{}:{}}}",
183 op_idx[&operand_idx], modifier
184 ));
185 } else {
186 template_str.push_str(&format!("${{{}}}", op_idx[&operand_idx]));
187 }
188 }
189 InlineAsmOperandRef::Const { ref string } => {
190 // Const operands get injected directly into the template
191 template_str.push_str(string);
192 }
193 InlineAsmOperandRef::SymFn { .. }
194 | InlineAsmOperandRef::SymStatic { .. } => {
195 // Only emit the raw symbol name
196 template_str.push_str(&format!("${{{}:c}}", op_idx[&operand_idx]));
197 }
198 }
199 }
200 }
201 }
202
cdc7bbd5 203 constraints.append(&mut clobbers);
f9f354fc
XL		 204 if !options.contains(InlineAsmOptions::PRESERVES_FLAGS) {
205 match asm_arch {
206 InlineAsmArch::AArch64 | InlineAsmArch::Arm => {
207 constraints.push("~{cc}".to_string());
208 }
209 InlineAsmArch::X86 | InlineAsmArch::X86_64 => {
210 constraints.extend_from_slice(&[
211 "~{dirflag}".to_string(),
212 "~{fpsr}".to_string(),
213 "~{flags}".to_string(),
214 ]);
215 }
94222f64
XL		 216 InlineAsmArch::RiscV32 | InlineAsmArch::RiscV64 => {
217 constraints.extend_from_slice(&[
218 "~{vtype}".to_string(),
219 "~{vl}".to_string(),
220 "~{vxsat}".to_string(),
221 "~{vxrm}".to_string(),
222 ]);
223 }
a2a8927a
XL		 224 InlineAsmArch::Avr => {
225 constraints.push("~{sreg}".to_string());
226 }
f9f354fc 227 InlineAsmArch::Nvptx64 => {}
17df50a5 228 InlineAsmArch::PowerPC | InlineAsmArch::PowerPC64 => {}
f035d41b 229 InlineAsmArch::Hexagon => {}
29967ef6 230 InlineAsmArch::Mips | InlineAsmArch::Mips64 => {}
94222f64 231 InlineAsmArch::S390x => {}
29967ef6 232 InlineAsmArch::SpirV => {}
3c0e092e 233 InlineAsmArch::Wasm32 | InlineAsmArch::Wasm64 => {}
17df50a5 234 InlineAsmArch::Bpf => {}
5099ac24
FG		 235 InlineAsmArch::Msp430 => {
236 constraints.push("~{sr}".to_string());
237 }
f9f354fc
XL		 238 }
239 }
240 if !options.contains(InlineAsmOptions::NOMEM) {
241 // This is actually ignored by LLVM, but it's probably best to keep
242 // it just in case. LLVM instead uses the ReadOnly/ReadNone
243 // attributes on the call instruction to optimize.
244 constraints.push("~{memory}".to_string());
245 }
246 let volatile = !options.contains(InlineAsmOptions::PURE);
247 let alignstack = !options.contains(InlineAsmOptions::NOSTACK);
248 let output_type = match &output_types[..] {
249 [] => self.type_void(),
250 [ty] => ty,
c295e0f8 251 tys => self.type_struct(tys, false),
f9f354fc
XL		 252 };
253 let dialect = match asm_arch {
254 InlineAsmArch::X86 | InlineAsmArch::X86_64
255 if !options.contains(InlineAsmOptions::ATT_SYNTAX) =>
256 {
5099ac24 257 llvm::AsmDialect::Intel
f9f354fc 258 }
5099ac24 259 _ => llvm::AsmDialect::Att,
f9f354fc
XL		 260 };
261 let result = inline_asm_call(
262 self,
263 &template_str,
264 &constraints.join(","),
265 &inputs,
266 output_type,
267 volatile,
268 alignstack,
269 dialect,
270 line_spans,
a2a8927a
XL		 271 options.contains(InlineAsmOptions::MAY_UNWIND),
272 dest_catch_funclet,
f9f354fc
XL		 273 )
274 .unwrap_or_else(|| span_bug!(line_spans[0], "LLVM asm constraint validation failed"));
275
5e7ed085 276 let mut attrs = SmallVec::<[_; 2]>::new();
f9f354fc
XL		 277 if options.contains(InlineAsmOptions::PURE) {
278 if options.contains(InlineAsmOptions::NOMEM) {
5e7ed085 279 attrs.push(llvm::AttributeKind::ReadNone.create_attr(self.cx.llcx));
f9f354fc 280 } else if options.contains(InlineAsmOptions::READONLY) {
5e7ed085 281 attrs.push(llvm::AttributeKind::ReadOnly.create_attr(self.cx.llcx));
f9f354fc 282 }
5e7ed085 283 attrs.push(llvm::AttributeKind::WillReturn.create_attr(self.cx.llcx));
29967ef6 284 } else if options.contains(InlineAsmOptions::NOMEM) {
5e7ed085 285 attrs.push(llvm::AttributeKind::InaccessibleMemOnly.create_attr(self.cx.llcx));
f9f354fc 286 } else {
29967ef6 287 // LLVM doesn't have an attribute to represent ReadOnly + SideEffect
f9f354fc 288 }
5e7ed085 289 attributes::apply_to_callsite(result, llvm::AttributePlace::Function, &{ attrs });
f9f354fc 290
04454e1e
FG		 291 // Switch to the 'normal' basic block if we did an `invoke` instead of a `call`
292 if let Some((dest, _, _)) = dest_catch_funclet {
293 self.switch_to_block(dest);
294 }
295
f9f354fc
XL		 296 // Write results to outputs
297 for (idx, op) in operands.iter().enumerate() {
298 if let InlineAsmOperandRef::Out { reg, place: Some(place), .. }
299 | InlineAsmOperandRef::InOut { reg, out_place: Some(place), .. } = *op
300 {
301 let value = if output_types.len() == 1 {
302 result
303 } else {
304 self.extract_value(result, op_idx[&idx] as u64)
305 };
306 let value = llvm_fixup_output(self, value, reg.reg_class(), &place.layout);
307 OperandValue::Immediate(value).store(self, place);
308 }
309 }
a1dfa0c6 310 }
54a0048b 311}
cc61c64b 312
04454e1e 313impl<'tcx> AsmMethods<'tcx> for CodegenCx<'_, 'tcx> {
17df50a5
XL		 314 fn codegen_global_asm(
315 &self,
316 template: &[InlineAsmTemplatePiece],
04454e1e 317 operands: &[GlobalAsmOperandRef<'tcx>],
17df50a5
XL		 318 options: InlineAsmOptions,
319 _line_spans: &[Span],
320 ) {
321 let asm_arch = self.tcx.sess.asm_arch.unwrap();
322
323 // Default to Intel syntax on x86
324 let intel_syntax = matches!(asm_arch, InlineAsmArch::X86 | InlineAsmArch::X86_64)
325 && !options.contains(InlineAsmOptions::ATT_SYNTAX);
326
327 // Build the template string
328 let mut template_str = String::new();
329 if intel_syntax {
330 template_str.push_str(".intel_syntax\n");
331 }
332 for piece in template {
333 match *piece {
334 InlineAsmTemplatePiece::String(ref s) => template_str.push_str(s),
335 InlineAsmTemplatePiece::Placeholder { operand_idx, modifier: _, span: _ } => {
336 match operands[operand_idx] {
337 GlobalAsmOperandRef::Const { ref string } => {
338 // Const operands get injected directly into the
339 // template. Note that we don't need to escape $
340 // here unlike normal inline assembly.
341 template_str.push_str(string);
342 }
04454e1e
FG		 343 GlobalAsmOperandRef::SymFn { instance } => {
344 let llval = self.get_fn(instance);
345 self.add_compiler_used_global(llval);
346 let symbol = llvm::build_string(|s| unsafe {
347 llvm::LLVMRustGetMangledName(llval, s);
348 })
349 .expect("symbol is not valid UTF-8");
350 template_str.push_str(&symbol);
351 }
352 GlobalAsmOperandRef::SymStatic { def_id } => {
353 let llval = self
354 .renamed_statics
355 .borrow()
356 .get(&def_id)
357 .copied()
358 .unwrap_or_else(|| self.get_static(def_id));
359 self.add_compiler_used_global(llval);
360 let symbol = llvm::build_string(|s| unsafe {
361 llvm::LLVMRustGetMangledName(llval, s);
362 })
363 .expect("symbol is not valid UTF-8");
364 template_str.push_str(&symbol);
365 }
17df50a5
XL		 366 }
367 }
368 }
369 }
370 if intel_syntax {
371 template_str.push_str("\n.att_syntax\n");
372 }
373
a1dfa0c6 374 unsafe {
17df50a5
XL		 375 llvm::LLVMRustAppendModuleInlineAsm(
376 self.llmod,
377 template_str.as_ptr().cast(),
378 template_str.len(),
379 );
a1dfa0c6 380 }
cc61c64b
XL		 381 }
382}
532ac7d7 383
a2a8927a
XL		 384pub(crate) fn inline_asm_call<'ll>(
385 bx: &mut Builder<'_, 'll, '_>,
ba9703b0
XL		 386 asm: &str,
387 cons: &str,
532ac7d7
XL		 388 inputs: &[&'ll Value],
389 output: &'ll llvm::Type,
390 volatile: bool,
391 alignstack: bool,
5099ac24 392 dia: llvm::AsmDialect,
f9f354fc 393 line_spans: &[Span],
a2a8927a
XL		 394 unwind: bool,
395 dest_catch_funclet: Option<(
396 &'ll llvm::BasicBlock,
397 &'ll llvm::BasicBlock,
398 Option<&Funclet<'ll>>,
399 )>,
532ac7d7 400) -> Option<&'ll Value> {
dfeec247
XL		 401 let volatile = if volatile { llvm::True } else { llvm::False };
402 let alignstack = if alignstack { llvm::True } else { llvm::False };
a2a8927a 403 let can_throw = if unwind { llvm::True } else { llvm::False };
dfeec247
XL		 404
405 let argtys = inputs
406 .iter()
407 .map(|v| {
408 debug!("Asm Input Type: {:?}", *v);
409 bx.cx.val_ty(*v)
410 })
411 .collect::<Vec<_>>();
532ac7d7
XL		 412
413 debug!("Asm Output Type: {:?}", output);
a2a8927a 414 let fty = bx.cx.type_func(&argtys, output);
532ac7d7
XL		 415 unsafe {
416 // Ask LLVM to verify that the constraints are well-formed.
ba9703b0 417 let constraints_ok = llvm::LLVMRustInlineAsmVerify(fty, cons.as_ptr().cast(), cons.len());
416331ca 418 debug!("constraint verification result: {:?}", constraints_ok);
532ac7d7
XL		 419 if constraints_ok {
420 let v = llvm::LLVMRustInlineAsm(
421 fty,
ba9703b0
XL		 422 asm.as_ptr().cast(),
423 asm.len(),
424 cons.as_ptr().cast(),
425 cons.len(),
532ac7d7
XL		 426 volatile,
427 alignstack,
5099ac24 428 dia,
a2a8927a 429 can_throw,
532ac7d7 430 );
a2a8927a
XL		 431
432 let call = if let Some((dest, catch, funclet)) = dest_catch_funclet {
2b03887a 433 bx.invoke(fty, None, v, inputs, dest, catch, funclet)
a2a8927a 434 } else {
2b03887a 435 bx.call(fty, None, v, inputs, None)
a2a8927a 436 };
f9f354fc
XL		 437
438 // Store mark in a metadata node so we can map LLVM errors
439 // back to source locations. See #17552.
440 let key = "srcloc";
441 let kind = llvm::LLVMGetMDKindIDInContext(
442 bx.llcx,
443 key.as_ptr() as *const c_char,
444 key.len() as c_uint,
445 );
446
447 // srcloc contains one integer for each line of assembly code.
448 // Unfortunately this isn't enough to encode a full span so instead
449 // we just encode the start position of each line.
450 // FIXME: Figure out a way to pass the entire line spans.
451 let mut srcloc = vec![];
5099ac24 452 if dia == llvm::AsmDialect::Intel && line_spans.len() > 1 {
f9f354fc
XL		 453 // LLVM inserts an extra line to add the ".intel_syntax", so add
454 // a dummy srcloc entry for it.
455 //
456 // Don't do this if we only have 1 line span since that may be
457 // due to the asm template string coming from a macro. LLVM will
458 // default to the first srcloc for lines that don't have an
459 // associated srcloc.
460 srcloc.push(bx.const_i32(0));
461 }
462 srcloc.extend(line_spans.iter().map(|span| bx.const_i32(span.lo().to_u32() as i32)));
463 let md = llvm::LLVMMDNodeInContext(bx.llcx, srcloc.as_ptr(), srcloc.len() as u32);
464 llvm::LLVMSetMetadata(call, kind, md);
465
466 Some(call)
532ac7d7
XL		 467 } else {
468 // LLVM has detected an issue with our constraints, bail out
469 None
470 }
471 }
472}
f9f354fc 473
3dfed10e
XL		 474/// If the register is an xmm/ymm/zmm register then return its index.
475fn xmm_reg_index(reg: InlineAsmReg) -> Option<u32> {
476 match reg {
477 InlineAsmReg::X86(reg)
478 if reg as u32 >= X86InlineAsmReg::xmm0 as u32
479 && reg as u32 <= X86InlineAsmReg::xmm15 as u32 =>
480 {
481 Some(reg as u32 - X86InlineAsmReg::xmm0 as u32)
482 }
483 InlineAsmReg::X86(reg)
484 if reg as u32 >= X86InlineAsmReg::ymm0 as u32
485 && reg as u32 <= X86InlineAsmReg::ymm15 as u32 =>
486 {
487 Some(reg as u32 - X86InlineAsmReg::ymm0 as u32)
488 }
489 InlineAsmReg::X86(reg)
490 if reg as u32 >= X86InlineAsmReg::zmm0 as u32
491 && reg as u32 <= X86InlineAsmReg::zmm31 as u32 =>
492 {
493 Some(reg as u32 - X86InlineAsmReg::zmm0 as u32)
494 }
495 _ => None,
496 }
497}
498
499/// If the register is an AArch64 vector register then return its index.
500fn a64_vreg_index(reg: InlineAsmReg) -> Option<u32> {
501 match reg {
502 InlineAsmReg::AArch64(reg)
503 if reg as u32 >= AArch64InlineAsmReg::v0 as u32
504 && reg as u32 <= AArch64InlineAsmReg::v31 as u32 =>
505 {
506 Some(reg as u32 - AArch64InlineAsmReg::v0 as u32)
507 }
508 _ => None,
509 }
510}
511
f9f354fc 512/// Converts a register class to an LLVM constraint code.
a2a8927a 513fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> String {
f9f354fc 514 match reg {
3dfed10e
XL		 515 // For vector registers LLVM wants the register name to match the type size.
516 InlineAsmRegOrRegClass::Reg(reg) => {
517 if let Some(idx) = xmm_reg_index(reg) {
518 let class = if let Some(layout) = layout {
519 match layout.size.bytes() {
520 64 => 'z',
521 32 => 'y',
522 _ => 'x',
523 }
524 } else {
525 // We use f32 as the type for discarded outputs
526 'x'
527 };
528 format!("{{{}mm{}}}", class, idx)
529 } else if let Some(idx) = a64_vreg_index(reg) {
530 let class = if let Some(layout) = layout {
531 match layout.size.bytes() {
532 16 => 'q',
533 8 => 'd',
534 4 => 's',
535 2 => 'h',
536 1 => 'd', // We fixup i8 to i8x8
537 _ => unreachable!(),
538 }
539 } else {
540 // We use i64x2 as the type for discarded outputs
541 'q'
542 };
543 format!("{{{}{}}}", class, idx)
544 } else if reg == InlineAsmReg::AArch64(AArch64InlineAsmReg::x30) {
545 // LLVM doesn't recognize x30
1b1a35ee 546 "{lr}".to_string()
6a06907d
XL		 547 } else if reg == InlineAsmReg::Arm(ArmInlineAsmReg::r14) {
548 // LLVM doesn't recognize r14
549 "{lr}".to_string()
3dfed10e
XL		 550 } else {
551 format!("{{{}}}", reg.name())
552 }
553 }
2b03887a
FG		 554 // The constraints can be retrieved from
555 // https://llvm.org/docs/LangRef.html#supported-constraint-code-list
f9f354fc
XL		 556 InlineAsmRegOrRegClass::RegClass(reg) => match reg {
557 InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => "r",
558 InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) => "w",
559 InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => "x",
136023e0
XL		 560 InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => {
561 unreachable!("clobber-only")
562 }
f9f354fc 563 InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) => "r",
f9f354fc
XL		 564 InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg)
565 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16)
566 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8) => "t",
567 InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16)
568 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8)
569 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4) => "x",
570 InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg)
571 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg) => "w",
f035d41b 572 InlineAsmRegClass::Hexagon(HexagonInlineAsmRegClass::reg) => "r",
1b1a35ee
XL		 573 InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg) => "r",
574 InlineAsmRegClass::Mips(MipsInlineAsmRegClass::freg) => "f",
f9f354fc
XL		 575 InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg16) => "h",
576 InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg32) => "r",
577 InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg64) => "l",
17df50a5
XL		 578 InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg) => "r",
579 InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => "b",
580 InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::freg) => "f",
94222f64
XL		 581 InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::cr)
582 | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::xer) => {
583 unreachable!("clobber-only")
584 }
f9f354fc
XL		 585 InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) => "r",
586 InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => "f",
136023e0
XL		 587 InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::vreg) => {
588 unreachable!("clobber-only")
589 }
f9f354fc
XL		 590 InlineAsmRegClass::X86(X86InlineAsmRegClass::reg) => "r",
591 InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => "Q",
592 InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_byte) => "q",
593 InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg)
594 | InlineAsmRegClass::X86(X86InlineAsmRegClass::ymm_reg) => "x",
595 InlineAsmRegClass::X86(X86InlineAsmRegClass::zmm_reg) => "v",
596 InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => "^Yk",
136023e0 597 InlineAsmRegClass::X86(
04454e1e
FG		 598 X86InlineAsmRegClass::x87_reg
599 | X86InlineAsmRegClass::mmx_reg
923072b8
FG		 600 | X86InlineAsmRegClass::kreg0
601 | X86InlineAsmRegClass::tmm_reg,
136023e0 602 ) => unreachable!("clobber-only"),
fc512014 603 InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => "r",
17df50a5
XL		 604 InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::reg) => "r",
605 InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::wreg) => "w",
a2a8927a
XL		 606 InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg) => "r",
607 InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_upper) => "d",
608 InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_pair) => "r",
609 InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_iw) => "w",
610 InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_ptr) => "e",
94222f64
XL		 611 InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg) => "r",
612 InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => "f",
5099ac24 613 InlineAsmRegClass::Msp430(Msp430InlineAsmRegClass::reg) => "r",
29967ef6
XL		 614 InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => {
615 bug!("LLVM backend does not support SPIR-V")
616 }
6a06907d 617 InlineAsmRegClass::Err => unreachable!(),
f9f354fc
XL		 618 }
619 .to_string(),
620 }
621}
622
623/// Converts a modifier into LLVM's equivalent modifier.
624fn modifier_to_llvm(
625 arch: InlineAsmArch,
626 reg: InlineAsmRegClass,
627 modifier: Option<char>,
628) -> Option<char> {
2b03887a
FG		 629 // The modifiers can be retrieved from
630 // https://llvm.org/docs/LangRef.html#asm-template-argument-modifiers
f9f354fc
XL		 631 match reg {
632 InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => modifier,
633 InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg)
634 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => {
635 if modifier == Some('v') { None } else { modifier }
636 }
136023e0
XL		 637 InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => {
638 unreachable!("clobber-only")
639 }
a2a8927a 640 InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) => None,
f9f354fc
XL		 641 InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg)
642 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16) => None,
643 InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg)
644 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16)
645 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8) => Some('P'),
646 InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg)
647 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8)
648 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4) => {
649 if modifier.is_none() {
650 Some('q')
651 } else {
652 modifier
653 }
654 }
f035d41b 655 InlineAsmRegClass::Hexagon(_) => None,
1b1a35ee 656 InlineAsmRegClass::Mips(_) => None,
f9f354fc 657 InlineAsmRegClass::Nvptx(_) => None,
17df50a5 658 InlineAsmRegClass::PowerPC(_) => None,
f9f354fc
XL		 659 InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg)
660 | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => None,
136023e0
XL		 661 InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::vreg) => {
662 unreachable!("clobber-only")
663 }
f9f354fc
XL		 664 InlineAsmRegClass::X86(X86InlineAsmRegClass::reg)
665 | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => match modifier {
666 None if arch == InlineAsmArch::X86_64 => Some('q'),
667 None => Some('k'),
668 Some('l') => Some('b'),
669 Some('h') => Some('h'),
670 Some('x') => Some('w'),
671 Some('e') => Some('k'),
672 Some('r') => Some('q'),
673 _ => unreachable!(),
674 },
675 InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_byte) => None,
676 InlineAsmRegClass::X86(reg @ X86InlineAsmRegClass::xmm_reg)
677 | InlineAsmRegClass::X86(reg @ X86InlineAsmRegClass::ymm_reg)
678 | InlineAsmRegClass::X86(reg @ X86InlineAsmRegClass::zmm_reg) => match (reg, modifier) {
679 (X86InlineAsmRegClass::xmm_reg, None) => Some('x'),
680 (X86InlineAsmRegClass::ymm_reg, None) => Some('t'),
681 (X86InlineAsmRegClass::zmm_reg, None) => Some('g'),
682 (_, Some('x')) => Some('x'),
683 (_, Some('y')) => Some('t'),
684 (_, Some('z')) => Some('g'),
685 _ => unreachable!(),
686 },
687 InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => None,
04454e1e
FG		 688 InlineAsmRegClass::X86(
689 X86InlineAsmRegClass::x87_reg
690 | X86InlineAsmRegClass::mmx_reg
923072b8
FG		 691 | X86InlineAsmRegClass::kreg0
692 | X86InlineAsmRegClass::tmm_reg,
04454e1e 693 ) => {
136023e0
XL		 694 unreachable!("clobber-only")
695 }
fc512014 696 InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => None,
17df50a5 697 InlineAsmRegClass::Bpf(_) => None,
a2a8927a
XL		 698 InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_pair)
699 | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_iw)
700 | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_ptr) => match modifier {
701 Some('h') => Some('B'),
702 Some('l') => Some('A'),
703 _ => None,
704 },
705 InlineAsmRegClass::Avr(_) => None,
94222f64 706 InlineAsmRegClass::S390x(_) => None,
5099ac24 707 InlineAsmRegClass::Msp430(_) => None,
29967ef6
XL		 708 InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => {
709 bug!("LLVM backend does not support SPIR-V")
710 }
6a06907d 711 InlineAsmRegClass::Err => unreachable!(),
f9f354fc
XL		 712 }
713}
714
715/// Type to use for outputs that are discarded. It doesn't really matter what
716/// the type is, as long as it is valid for the constraint code.
a2a8927a 717fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &'ll Type {
f9f354fc
XL		 718 match reg {
719 InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => cx.type_i32(),
720 InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg)
721 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => {
722 cx.type_vector(cx.type_i64(), 2)
723 }
136023e0
XL		 724 InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => {
725 unreachable!("clobber-only")
726 }
a2a8927a 727 InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) => cx.type_i32(),
f9f354fc
XL		 728 InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg)
729 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16) => cx.type_f32(),
730 InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg)
731 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16)
732 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8) => cx.type_f64(),
733 InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg)
734 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8)
735 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4) => {
736 cx.type_vector(cx.type_i64(), 2)
737 }
f035d41b 738 InlineAsmRegClass::Hexagon(HexagonInlineAsmRegClass::reg) => cx.type_i32(),
1b1a35ee
XL		 739 InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg) => cx.type_i32(),
740 InlineAsmRegClass::Mips(MipsInlineAsmRegClass::freg) => cx.type_f32(),
f9f354fc
XL		 741 InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg16) => cx.type_i16(),
742 InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg32) => cx.type_i32(),
743 InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg64) => cx.type_i64(),
17df50a5
XL		 744 InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg) => cx.type_i32(),
745 InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => cx.type_i32(),
746 InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::freg) => cx.type_f64(),
94222f64
XL		 747 InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::cr)
748 | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::xer) => {
749 unreachable!("clobber-only")
750 }
f9f354fc
XL		 751 InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) => cx.type_i32(),
752 InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => cx.type_f32(),
136023e0
XL		 753 InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::vreg) => {
754 unreachable!("clobber-only")
755 }
f9f354fc
XL		 756 InlineAsmRegClass::X86(X86InlineAsmRegClass::reg)
757 | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => cx.type_i32(),
758 InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_byte) => cx.type_i8(),
759 InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg)
760 | InlineAsmRegClass::X86(X86InlineAsmRegClass::ymm_reg)
761 | InlineAsmRegClass::X86(X86InlineAsmRegClass::zmm_reg) => cx.type_f32(),
762 InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => cx.type_i16(),
04454e1e
FG		 763 InlineAsmRegClass::X86(
764 X86InlineAsmRegClass::x87_reg
765 | X86InlineAsmRegClass::mmx_reg
923072b8
FG		 766 | X86InlineAsmRegClass::kreg0
767 | X86InlineAsmRegClass::tmm_reg,
04454e1e 768 ) => {
136023e0
XL		 769 unreachable!("clobber-only")
770 }
fc512014 771 InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => cx.type_i32(),
17df50a5
XL		 772 InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::reg) => cx.type_i64(),
773 InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::wreg) => cx.type_i32(),
a2a8927a
XL		 774 InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg) => cx.type_i8(),
775 InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_upper) => cx.type_i8(),
776 InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_pair) => cx.type_i16(),
777 InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_iw) => cx.type_i16(),
778 InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_ptr) => cx.type_i16(),
94222f64
XL		 779 InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg) => cx.type_i32(),
780 InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => cx.type_f64(),
5099ac24 781 InlineAsmRegClass::Msp430(Msp430InlineAsmRegClass::reg) => cx.type_i16(),
29967ef6
XL		 782 InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => {
783 bug!("LLVM backend does not support SPIR-V")
784 }
6a06907d 785 InlineAsmRegClass::Err => unreachable!(),
f9f354fc
XL		 786 }
787}
788
789/// Helper function to get the LLVM type for a Scalar. Pointers are returned as
790/// the equivalent integer type.
a2a8927a 791fn llvm_asm_scalar_type<'ll>(cx: &CodegenCx<'ll, '_>, scalar: Scalar) -> &'ll Type {
04454e1e 792 match scalar.primitive() {
f9f354fc
XL		 793 Primitive::Int(Integer::I8, _) => cx.type_i8(),
794 Primitive::Int(Integer::I16, _) => cx.type_i16(),
795 Primitive::Int(Integer::I32, _) => cx.type_i32(),
796 Primitive::Int(Integer::I64, _) => cx.type_i64(),
797 Primitive::F32 => cx.type_f32(),
798 Primitive::F64 => cx.type_f64(),
799 Primitive::Pointer => cx.type_isize(),
800 _ => unreachable!(),
801 }
802}
803
804/// Fix up an input value to work around LLVM bugs.
a2a8927a
XL		 805fn llvm_fixup_input<'ll, 'tcx>(
806 bx: &mut Builder<'_, 'll, 'tcx>,
f9f354fc
XL		 807 mut value: &'ll Value,
808 reg: InlineAsmRegClass,
809 layout: &TyAndLayout<'tcx>,
810) -> &'ll Value {
c295e0f8 811 match (reg, layout.abi) {
f9f354fc 812 (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => {
04454e1e 813 if let Primitive::Int(Integer::I8, _) = s.primitive() {
f9f354fc
XL		 814 let vec_ty = bx.cx.type_vector(bx.cx.type_i8(), 8);
815 bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0))
816 } else {
817 value
818 }
819 }
820 (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) => {
821 let elem_ty = llvm_asm_scalar_type(bx.cx, s);
822 let count = 16 / layout.size.bytes();
823 let vec_ty = bx.cx.type_vector(elem_ty, count);
04454e1e 824 if let Primitive::Pointer = s.primitive() {
f9f354fc
XL		 825 value = bx.ptrtoint(value, bx.cx.type_isize());
826 }
827 bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0))
828 }
829 (
830 InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16),
831 Abi::Vector { element, count },
832 ) if layout.size.bytes() == 8 => {
833 let elem_ty = llvm_asm_scalar_type(bx.cx, element);
c295e0f8 834 let vec_ty = bx.cx.type_vector(elem_ty, count);
f9f354fc
XL		 835 let indices: Vec<_> = (0..count * 2).map(|x| bx.const_i32(x as i32)).collect();
836 bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices))
837 }
838 (InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd), Abi::Scalar(s))
04454e1e 839 if s.primitive() == Primitive::F64 =>
f9f354fc
XL		 840 {
841 bx.bitcast(value, bx.cx.type_i64())
842 }
843 (
844 InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg),
845 Abi::Vector { .. },
846 ) if layout.size.bytes() == 64 => bx.bitcast(value, bx.cx.type_vector(bx.cx.type_f64(), 8)),
3dfed10e
XL		 847 (
848 InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16),
849 Abi::Scalar(s),
850 ) => {
04454e1e 851 if let Primitive::Int(Integer::I32, _) = s.primitive() {
3dfed10e
XL		 852 bx.bitcast(value, bx.cx.type_f32())
853 } else {
854 value
855 }
856 }
f9f354fc
XL		 857 (
858 InlineAsmRegClass::Arm(
3dfed10e 859 ArmInlineAsmRegClass::dreg
f9f354fc 860 | ArmInlineAsmRegClass::dreg_low8
3dfed10e 861 | ArmInlineAsmRegClass::dreg_low16,
f9f354fc
XL		 862 ),
863 Abi::Scalar(s),
864 ) => {
04454e1e 865 if let Primitive::Int(Integer::I64, _) = s.primitive() {
3dfed10e 866 bx.bitcast(value, bx.cx.type_f64())
f9f354fc
XL		 867 } else {
868 value
869 }
870 }
04454e1e
FG		 871 (InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg), Abi::Scalar(s)) => {
872 match s.primitive() {
873 // MIPS only supports register-length arithmetics.
874 Primitive::Int(Integer::I8 | Integer::I16, _) => bx.zext(value, bx.cx.type_i32()),
875 Primitive::F32 => bx.bitcast(value, bx.cx.type_i32()),
876 Primitive::F64 => bx.bitcast(value, bx.cx.type_i64()),
877 _ => value,
878 }
879 }
f9f354fc
XL		 880 _ => value,
881 }
882}
883
884/// Fix up an output value to work around LLVM bugs.
a2a8927a
XL		 885fn llvm_fixup_output<'ll, 'tcx>(
886 bx: &mut Builder<'_, 'll, 'tcx>,
f9f354fc
XL		 887 mut value: &'ll Value,
888 reg: InlineAsmRegClass,
889 layout: &TyAndLayout<'tcx>,
890) -> &'ll Value {
c295e0f8 891 match (reg, layout.abi) {
f9f354fc 892 (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => {
04454e1e 893 if let Primitive::Int(Integer::I8, _) = s.primitive() {
f9f354fc
XL		 894 bx.extract_element(value, bx.const_i32(0))
895 } else {
896 value
897 }
898 }
899 (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) => {
900 value = bx.extract_element(value, bx.const_i32(0));
04454e1e 901 if let Primitive::Pointer = s.primitive() {
f9f354fc
XL		 902 value = bx.inttoptr(value, layout.llvm_type(bx.cx));
903 }
904 value
905 }
906 (
907 InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16),
908 Abi::Vector { element, count },
909 ) if layout.size.bytes() == 8 => {
910 let elem_ty = llvm_asm_scalar_type(bx.cx, element);
c295e0f8
XL		 911 let vec_ty = bx.cx.type_vector(elem_ty, count * 2);
912 let indices: Vec<_> = (0..count).map(|x| bx.const_i32(x as i32)).collect();
f9f354fc
XL		 913 bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices))
914 }
915 (InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd), Abi::Scalar(s))
04454e1e 916 if s.primitive() == Primitive::F64 =>
f9f354fc
XL		 917 {
918 bx.bitcast(value, bx.cx.type_f64())
919 }
920 (
921 InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg),
922 Abi::Vector { .. },
923 ) if layout.size.bytes() == 64 => bx.bitcast(value, layout.llvm_type(bx.cx)),
3dfed10e
XL		 924 (
925 InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16),
926 Abi::Scalar(s),
927 ) => {
04454e1e 928 if let Primitive::Int(Integer::I32, _) = s.primitive() {
3dfed10e
XL		 929 bx.bitcast(value, bx.cx.type_i32())
930 } else {
931 value
932 }
933 }
f9f354fc
XL		 934 (
935 InlineAsmRegClass::Arm(
3dfed10e 936 ArmInlineAsmRegClass::dreg
f9f354fc 937 | ArmInlineAsmRegClass::dreg_low8
3dfed10e 938 | ArmInlineAsmRegClass::dreg_low16,
f9f354fc
XL		 939 ),
940 Abi::Scalar(s),
941 ) => {
04454e1e 942 if let Primitive::Int(Integer::I64, _) = s.primitive() {
3dfed10e 943 bx.bitcast(value, bx.cx.type_i64())
f9f354fc
XL		 944 } else {
945 value
946 }
947 }
04454e1e
FG		 948 (InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg), Abi::Scalar(s)) => {
949 match s.primitive() {
950 // MIPS only supports register-length arithmetics.
951 Primitive::Int(Integer::I8, _) => bx.trunc(value, bx.cx.type_i8()),
952 Primitive::Int(Integer::I16, _) => bx.trunc(value, bx.cx.type_i16()),
953 Primitive::F32 => bx.bitcast(value, bx.cx.type_f32()),
954 Primitive::F64 => bx.bitcast(value, bx.cx.type_f64()),
955 _ => value,
956 }
957 }
f9f354fc
XL		 958 _ => value,
959 }
960}
961
962/// Output type to use for llvm_fixup_output.
a2a8927a 963fn llvm_fixup_output_type<'ll, 'tcx>(
f9f354fc
XL		 964 cx: &CodegenCx<'ll, 'tcx>,
965 reg: InlineAsmRegClass,
966 layout: &TyAndLayout<'tcx>,
967) -> &'ll Type {
c295e0f8 968 match (reg, layout.abi) {
f9f354fc 969 (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => {
04454e1e 970 if let Primitive::Int(Integer::I8, _) = s.primitive() {
f9f354fc
XL		 971 cx.type_vector(cx.type_i8(), 8)
972 } else {
973 layout.llvm_type(cx)
974 }
975 }
976 (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) => {
977 let elem_ty = llvm_asm_scalar_type(cx, s);
978 let count = 16 / layout.size.bytes();
979 cx.type_vector(elem_ty, count)
980 }
981 (
982 InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16),
983 Abi::Vector { element, count },
984 ) if layout.size.bytes() == 8 => {
985 let elem_ty = llvm_asm_scalar_type(cx, element);
986 cx.type_vector(elem_ty, count * 2)
987 }
988 (InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd), Abi::Scalar(s))
04454e1e 989 if s.primitive() == Primitive::F64 =>
f9f354fc
XL		 990 {
991 cx.type_i64()
992 }
993 (
994 InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg),
995 Abi::Vector { .. },
996 ) if layout.size.bytes() == 64 => cx.type_vector(cx.type_f64(), 8),
3dfed10e
XL		 997 (
998 InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16),
999 Abi::Scalar(s),
1000 ) => {
04454e1e 1001 if let Primitive::Int(Integer::I32, _) = s.primitive() {
3dfed10e
XL		 1002 cx.type_f32()
1003 } else {
1004 layout.llvm_type(cx)
1005 }
1006 }
f9f354fc
XL		 1007 (
1008 InlineAsmRegClass::Arm(
3dfed10e 1009 ArmInlineAsmRegClass::dreg
f9f354fc 1010 | ArmInlineAsmRegClass::dreg_low8
3dfed10e 1011 | ArmInlineAsmRegClass::dreg_low16,
f9f354fc
XL		 1012 ),
1013 Abi::Scalar(s),
1014 ) => {
04454e1e 1015 if let Primitive::Int(Integer::I64, _) = s.primitive() {
3dfed10e 1016 cx.type_f64()
f9f354fc
XL		 1017 } else {
1018 layout.llvm_type(cx)
1019 }
1020 }
04454e1e
FG		 1021 (InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg), Abi::Scalar(s)) => {
1022 match s.primitive() {
1023 // MIPS only supports register-length arithmetics.
1024 Primitive::Int(Integer::I8 | Integer::I16, _) => cx.type_i32(),
1025 Primitive::F32 => cx.type_i32(),
1026 Primitive::F64 => cx.type_i64(),
1027 _ => layout.llvm_type(cx),
1028 }
1029 }
f9f354fc
XL		 1030 _ => layout.llvm_type(cx),
1031 }
1032}
backport for Debian buster