1 use crate::builder
::Builder
;
2 use crate::common
::Funclet
;
3 use crate::context
::CodegenCx
;
6 use crate::type_
::Type
;
7 use crate::type_of
::LayoutLlvmExt
;
8 use crate::value
::Value
;
10 use rustc_ast
::LlvmAsmDialect
;
11 use rustc_ast
::{InlineAsmOptions, InlineAsmTemplatePiece}
;
12 use rustc_codegen_ssa
::mir
::operand
::OperandValue
;
13 use rustc_codegen_ssa
::mir
::place
::PlaceRef
;
14 use rustc_codegen_ssa
::traits
::*;
15 use rustc_data_structures
::fx
::FxHashMap
;
17 use rustc_middle
::ty
::layout
::TyAndLayout
;
18 use rustc_middle
::{bug, span_bug, ty::Instance}
;
19 use rustc_span
::{Pos, Span, Symbol}
;
20 use rustc_target
::abi
::*;
21 use rustc_target
::asm
::*;
23 use libc
::{c_char, c_uint}
;
26 impl<'ll
, 'tcx
> AsmBuilderMethods
<'tcx
> for Builder
<'_
, 'll
, 'tcx
> {
27 fn codegen_llvm_inline_asm(
29 ia
: &hir
::LlvmInlineAsmInner
,
30 outputs
: Vec
<PlaceRef
<'tcx
, &'ll Value
>>,
31 mut inputs
: Vec
<&'ll Value
>,
34 let mut ext_constraints
= vec
![];
35 let mut output_types
= vec
![];
37 // Prepare the output operands
38 let mut indirect_outputs
= vec
![];
39 for (i
, (out
, &place
)) in ia
.outputs
.iter().zip(&outputs
).enumerate() {
41 let operand
= self.load_operand(place
);
42 if let OperandValue
::Immediate(_
) = operand
.val
{
43 inputs
.push(operand
.immediate());
45 ext_constraints
.push(i
.to_string());
48 let operand
= self.load_operand(place
);
49 if let OperandValue
::Immediate(_
) = operand
.val
{
50 indirect_outputs
.push(operand
.immediate());
53 output_types
.push(place
.layout
.llvm_type(self.cx
));
56 if !indirect_outputs
.is_empty() {
57 indirect_outputs
.extend_from_slice(&inputs
);
58 inputs
= indirect_outputs
;
61 let clobbers
= ia
.clobbers
.iter().map(|s
| format
!("~{{{}}}", &s
));
63 // Default per-arch clobbers
64 // Basically what clang does
65 let arch_clobbers
= match &self.sess().target
.arch
[..] {
66 "x86" | "x86_64" => &["~{dirflag}", "~{fpsr}", "~{flags}"][..],
67 "mips" | "mips64" => &["~{$1}"],
71 let all_constraints
= ia
74 .map(|out
| out
.constraint
.to_string())
75 .chain(ia
.inputs
.iter().map(|s
| s
.to_string()))
76 .chain(ext_constraints
)
78 .chain(arch_clobbers
.iter().map(|s
| (*s
).to_string()))
79 .collect
::<Vec
<String
>>()
82 debug
!("Asm Constraints: {}", &all_constraints
);
84 // Depending on how many outputs we have, the return type is different
85 let num_outputs
= output_types
.len();
86 let output_type
= match num_outputs
{
87 0 => self.type_void(),
89 _
=> self.type_struct(&output_types
, false),
92 let asm
= ia
.asm
.as_str();
93 let r
= inline_asm_call(
111 // Again, based on how many outputs we have
112 let outputs
= ia
.outputs
.iter().zip(&outputs
).filter(|&(o
, _
)| !o
.is_indirect
);
113 for (i
, (_
, &place
)) in outputs
.enumerate() {
114 let v
= if num_outputs
== 1 { r }
else { self.extract_value(r, i as u64) }
;
115 OperandValue
::Immediate(v
).store(self, place
);
121 fn codegen_inline_asm(
123 template
: &[InlineAsmTemplatePiece
],
124 operands
: &[InlineAsmOperandRef
<'tcx
, Self>],
125 options
: InlineAsmOptions
,
127 instance
: Instance
<'_
>,
128 dest_catch_funclet
: Option
<(Self::BasicBlock
, Self::BasicBlock
, Option
<&Self::Funclet
>)>,
130 let asm_arch
= self.tcx
.sess
.asm_arch
.unwrap();
132 // Collect the types of output operands
133 let mut constraints
= vec
![];
134 let mut clobbers
= vec
![];
135 let mut output_types
= vec
![];
136 let mut op_idx
= FxHashMap
::default();
137 let mut clobbered_x87
= false;
138 for (idx
, op
) in operands
.iter().enumerate() {
140 InlineAsmOperandRef
::Out { reg, late, place }
=> {
141 let is_target_supported
= |reg_class
: InlineAsmRegClass
| {
142 for &(_
, feature
) in reg_class
.supported_types(asm_arch
) {
143 if let Some(feature
) = feature
{
144 let codegen_fn_attrs
= self.tcx
.codegen_fn_attrs(instance
.def_id());
145 let feature_name
= Symbol
::intern(feature
);
146 if self.tcx
.sess
.target_features
.contains(&feature_name
)
147 || codegen_fn_attrs
.target_features
.contains(&feature_name
)
152 // Register class is unconditionally supported
159 let mut layout
= None
;
160 let ty
= if let Some(ref place
) = place
{
161 layout
= Some(&place
.layout
);
162 llvm_fixup_output_type(self.cx
, reg
.reg_class(), &place
.layout
)
165 InlineAsmRegClass
::X86(
166 X86InlineAsmRegClass
::mmx_reg
| X86InlineAsmRegClass
::x87_reg
169 // Special handling for x87/mmx registers: we always
170 // clobber the whole set if one register is marked as
171 // clobbered. This is due to the way LLVM handles the
172 // FP stack in inline assembly.
174 clobbered_x87
= true;
175 clobbers
.push("~{st}".to_string());
177 clobbers
.push(format
!("~{{st({})}}", i
));
181 } else if !is_target_supported(reg
.reg_class())
182 || reg
.reg_class().is_clobber_only(asm_arch
)
184 // We turn discarded outputs into clobber constraints
185 // if the target feature needed by the register class is
186 // disabled. This is necessary otherwise LLVM will try
187 // to actually allocate a register for the dummy output.
188 assert
!(matches
!(reg
, InlineAsmRegOrRegClass
::Reg(_
)));
189 clobbers
.push(format
!("~{}", reg_to_llvm(reg
, None
)));
192 // If the output is discarded, we don't really care what
193 // type is used. We're just using this to tell LLVM to
194 // reserve the register.
195 dummy_output_type(self.cx
, reg
.reg_class())
197 output_types
.push(ty
);
198 op_idx
.insert(idx
, constraints
.len());
199 let prefix
= if late { "=" }
else { "=&" }
;
200 constraints
.push(format
!("{}{}", prefix
, reg_to_llvm(reg
, layout
)));
202 InlineAsmOperandRef
::InOut { reg, late, in_value, out_place }
=> {
203 let layout
= if let Some(ref out_place
) = out_place
{
206 // LLVM required tied operands to have the same type,
207 // so we just use the type of the input.
210 let ty
= llvm_fixup_output_type(self.cx
, reg
.reg_class(), layout
);
211 output_types
.push(ty
);
212 op_idx
.insert(idx
, constraints
.len());
213 let prefix
= if late { "=" }
else { "=&" }
;
214 constraints
.push(format
!("{}{}", prefix
, reg_to_llvm(reg
, Some(layout
))));
220 // Collect input operands
221 let mut inputs
= vec
![];
222 for (idx
, op
) in operands
.iter().enumerate() {
224 InlineAsmOperandRef
::In { reg, value }
=> {
226 llvm_fixup_input(self, value
.immediate(), reg
.reg_class(), &value
.layout
);
228 op_idx
.insert(idx
, constraints
.len());
229 constraints
.push(reg_to_llvm(reg
, Some(&value
.layout
)));
231 InlineAsmOperandRef
::InOut { reg, late: _, in_value, out_place: _ }
=> {
232 let value
= llvm_fixup_input(
234 in_value
.immediate(),
239 constraints
.push(format
!("{}", op_idx
[&idx
]));
241 InlineAsmOperandRef
::SymFn { instance }
=> {
242 inputs
.push(self.cx
.get_fn(instance
));
243 op_idx
.insert(idx
, constraints
.len());
244 constraints
.push("s".to_string());
246 InlineAsmOperandRef
::SymStatic { def_id }
=> {
247 inputs
.push(self.cx
.get_static(def_id
));
248 op_idx
.insert(idx
, constraints
.len());
249 constraints
.push("s".to_string());
255 // Build the template string
256 let mut template_str
= String
::new();
257 for piece
in template
{
259 InlineAsmTemplatePiece
::String(ref s
) => {
263 template_str
.push_str("$$");
265 template_str
.push(c
);
269 template_str
.push_str(s
)
272 InlineAsmTemplatePiece
::Placeholder { operand_idx, modifier, span: _ }
=> {
273 match operands
[operand_idx
] {
274 InlineAsmOperandRef
::In { reg, .. }
275 | InlineAsmOperandRef
::Out { reg, .. }
276 | InlineAsmOperandRef
::InOut { reg, .. }
=> {
277 let modifier
= modifier_to_llvm(asm_arch
, reg
.reg_class(), modifier
);
278 if let Some(modifier
) = modifier
{
279 template_str
.push_str(&format
!(
281 op_idx
[&operand_idx
], modifier
284 template_str
.push_str(&format
!("${{{}}}", op_idx
[&operand_idx
]));
287 InlineAsmOperandRef
::Const { ref string }
=> {
288 // Const operands get injected directly into the template
289 template_str
.push_str(string
);
291 InlineAsmOperandRef
::SymFn { .. }
292 | InlineAsmOperandRef
::SymStatic { .. }
=> {
293 // Only emit the raw symbol name
294 template_str
.push_str(&format
!("${{{}:c}}", op_idx
[&operand_idx
]));
301 constraints
.append(&mut clobbers
);
302 if !options
.contains(InlineAsmOptions
::PRESERVES_FLAGS
) {
304 InlineAsmArch
::AArch64
| InlineAsmArch
::Arm
=> {
305 constraints
.push("~{cc}".to_string());
307 InlineAsmArch
::X86
| InlineAsmArch
::X86_64
=> {
308 constraints
.extend_from_slice(&[
309 "~{dirflag}".to_string(),
310 "~{fpsr}".to_string(),
311 "~{flags}".to_string(),
314 InlineAsmArch
::RiscV32
| InlineAsmArch
::RiscV64
=> {
315 constraints
.extend_from_slice(&[
316 "~{vtype}".to_string(),
318 "~{vxsat}".to_string(),
319 "~{vxrm}".to_string(),
322 InlineAsmArch
::Avr
=> {
323 constraints
.push("~{sreg}".to_string());
325 InlineAsmArch
::Nvptx64
=> {}
326 InlineAsmArch
::PowerPC
| InlineAsmArch
::PowerPC64
=> {}
327 InlineAsmArch
::Hexagon
=> {}
328 InlineAsmArch
::Mips
| InlineAsmArch
::Mips64
=> {}
329 InlineAsmArch
::S390x
=> {}
330 InlineAsmArch
::SpirV
=> {}
331 InlineAsmArch
::Wasm32
| InlineAsmArch
::Wasm64
=> {}
332 InlineAsmArch
::Bpf
=> {}
335 if !options
.contains(InlineAsmOptions
::NOMEM
) {
336 // This is actually ignored by LLVM, but it's probably best to keep
337 // it just in case. LLVM instead uses the ReadOnly/ReadNone
338 // attributes on the call instruction to optimize.
339 constraints
.push("~{memory}".to_string());
341 let volatile
= !options
.contains(InlineAsmOptions
::PURE
);
342 let alignstack
= !options
.contains(InlineAsmOptions
::NOSTACK
);
343 let output_type
= match &output_types
[..] {
344 [] => self.type_void(),
346 tys
=> self.type_struct(tys
, false),
348 let dialect
= match asm_arch
{
349 InlineAsmArch
::X86
| InlineAsmArch
::X86_64
350 if !options
.contains(InlineAsmOptions
::ATT_SYNTAX
) =>
352 LlvmAsmDialect
::Intel
354 _
=> LlvmAsmDialect
::Att
,
356 let result
= inline_asm_call(
359 &constraints
.join(","),
366 options
.contains(InlineAsmOptions
::MAY_UNWIND
),
369 .unwrap_or_else(|| span_bug
!(line_spans
[0], "LLVM asm constraint validation failed"));
371 if options
.contains(InlineAsmOptions
::PURE
) {
372 if options
.contains(InlineAsmOptions
::NOMEM
) {
373 llvm
::Attribute
::ReadNone
.apply_callsite(llvm
::AttributePlace
::Function
, result
);
374 } else if options
.contains(InlineAsmOptions
::READONLY
) {
375 llvm
::Attribute
::ReadOnly
.apply_callsite(llvm
::AttributePlace
::Function
, result
);
377 llvm
::Attribute
::WillReturn
.apply_callsite(llvm
::AttributePlace
::Function
, result
);
378 } else if options
.contains(InlineAsmOptions
::NOMEM
) {
379 llvm
::Attribute
::InaccessibleMemOnly
380 .apply_callsite(llvm
::AttributePlace
::Function
, result
);
382 // LLVM doesn't have an attribute to represent ReadOnly + SideEffect
385 // Write results to outputs
386 for (idx
, op
) in operands
.iter().enumerate() {
387 if let InlineAsmOperandRef
::Out { reg, place: Some(place), .. }
388 | InlineAsmOperandRef
::InOut { reg, out_place: Some(place), .. }
= *op
390 let value
= if output_types
.len() == 1 {
393 self.extract_value(result
, op_idx
[&idx
] as u64)
395 let value
= llvm_fixup_output(self, value
, reg
.reg_class(), &place
.layout
);
396 OperandValue
::Immediate(value
).store(self, place
);
402 impl AsmMethods
for CodegenCx
<'_
, '_
> {
403 fn codegen_global_asm(
405 template
: &[InlineAsmTemplatePiece
],
406 operands
: &[GlobalAsmOperandRef
],
407 options
: InlineAsmOptions
,
408 _line_spans
: &[Span
],
410 let asm_arch
= self.tcx
.sess
.asm_arch
.unwrap();
412 // Default to Intel syntax on x86
413 let intel_syntax
= matches
!(asm_arch
, InlineAsmArch
::X86
| InlineAsmArch
::X86_64
)
414 && !options
.contains(InlineAsmOptions
::ATT_SYNTAX
);
416 // Build the template string
417 let mut template_str
= String
::new();
419 template_str
.push_str(".intel_syntax\n");
421 for piece
in template
{
423 InlineAsmTemplatePiece
::String(ref s
) => template_str
.push_str(s
),
424 InlineAsmTemplatePiece
::Placeholder { operand_idx, modifier: _, span: _ }
=> {
425 match operands
[operand_idx
] {
426 GlobalAsmOperandRef
::Const { ref string }
=> {
427 // Const operands get injected directly into the
428 // template. Note that we don't need to escape $
429 // here unlike normal inline assembly.
430 template_str
.push_str(string
);
437 template_str
.push_str("\n.att_syntax\n");
441 llvm
::LLVMRustAppendModuleInlineAsm(
443 template_str
.as_ptr().cast(),
450 pub(crate) fn inline_asm_call
<'ll
>(
451 bx
: &mut Builder
<'_
, 'll
, '_
>,
454 inputs
: &[&'ll Value
],
455 output
: &'ll llvm
::Type
,
461 dest_catch_funclet
: Option
<(
462 &'ll llvm
::BasicBlock
,
463 &'ll llvm
::BasicBlock
,
464 Option
<&Funclet
<'ll
>>,
466 ) -> Option
<&'ll Value
> {
467 let volatile
= if volatile { llvm::True }
else { llvm::False }
;
468 let alignstack
= if alignstack { llvm::True }
else { llvm::False }
;
469 let can_throw
= if unwind { llvm::True }
else { llvm::False }
;
474 debug
!("Asm Input Type: {:?}", *v
);
477 .collect
::<Vec
<_
>>();
479 debug
!("Asm Output Type: {:?}", output
);
480 let fty
= bx
.cx
.type_func(&argtys
, output
);
482 // Ask LLVM to verify that the constraints are well-formed.
483 let constraints_ok
= llvm
::LLVMRustInlineAsmVerify(fty
, cons
.as_ptr().cast(), cons
.len());
484 debug
!("constraint verification result: {:?}", constraints_ok
);
486 if unwind
&& llvm_util
::get_version() < (13, 0, 0) {
487 bx
.cx
.sess().span_fatal(
489 "unwinding from inline assembly is only supported on llvm >= 13.",
493 let v
= llvm
::LLVMRustInlineAsm(
497 cons
.as_ptr().cast(),
501 llvm
::AsmDialect
::from_generic(dia
),
505 let call
= if let Some((dest
, catch, funclet
)) = dest_catch_funclet
{
506 bx
.invoke(fty
, v
, inputs
, dest
, catch, funclet
)
508 bx
.call(fty
, v
, inputs
, None
)
511 // Store mark in a metadata node so we can map LLVM errors
512 // back to source locations. See #17552.
514 let kind
= llvm
::LLVMGetMDKindIDInContext(
516 key
.as_ptr() as *const c_char
,
520 // srcloc contains one integer for each line of assembly code.
521 // Unfortunately this isn't enough to encode a full span so instead
522 // we just encode the start position of each line.
523 // FIXME: Figure out a way to pass the entire line spans.
524 let mut srcloc
= vec
![];
525 if dia
== LlvmAsmDialect
::Intel
&& line_spans
.len() > 1 {
526 // LLVM inserts an extra line to add the ".intel_syntax", so add
527 // a dummy srcloc entry for it.
529 // Don't do this if we only have 1 line span since that may be
530 // due to the asm template string coming from a macro. LLVM will
531 // default to the first srcloc for lines that don't have an
532 // associated srcloc.
533 srcloc
.push(bx
.const_i32(0));
535 srcloc
.extend(line_spans
.iter().map(|span
| bx
.const_i32(span
.lo().to_u32() as i32)));
536 let md
= llvm
::LLVMMDNodeInContext(bx
.llcx
, srcloc
.as_ptr(), srcloc
.len() as u32);
537 llvm
::LLVMSetMetadata(call
, kind
, md
);
541 // LLVM has detected an issue with our constraints, bail out
547 /// If the register is an xmm/ymm/zmm register then return its index.
548 fn xmm_reg_index(reg
: InlineAsmReg
) -> Option
<u32> {
550 InlineAsmReg
::X86(reg
)
551 if reg
as u32 >= X86InlineAsmReg
::xmm0
as u32
552 && reg
as u32 <= X86InlineAsmReg
::xmm15
as u32 =>
554 Some(reg
as u32 - X86InlineAsmReg
::xmm0
as u32)
556 InlineAsmReg
::X86(reg
)
557 if reg
as u32 >= X86InlineAsmReg
::ymm0
as u32
558 && reg
as u32 <= X86InlineAsmReg
::ymm15
as u32 =>
560 Some(reg
as u32 - X86InlineAsmReg
::ymm0
as u32)
562 InlineAsmReg
::X86(reg
)
563 if reg
as u32 >= X86InlineAsmReg
::zmm0
as u32
564 && reg
as u32 <= X86InlineAsmReg
::zmm31
as u32 =>
566 Some(reg
as u32 - X86InlineAsmReg
::zmm0
as u32)
572 /// If the register is an AArch64 vector register then return its index.
573 fn a64_vreg_index(reg
: InlineAsmReg
) -> Option
<u32> {
575 InlineAsmReg
::AArch64(reg
)
576 if reg
as u32 >= AArch64InlineAsmReg
::v0
as u32
577 && reg
as u32 <= AArch64InlineAsmReg
::v31
as u32 =>
579 Some(reg
as u32 - AArch64InlineAsmReg
::v0
as u32)
585 /// Converts a register class to an LLVM constraint code.
586 fn reg_to_llvm(reg
: InlineAsmRegOrRegClass
, layout
: Option
<&TyAndLayout
<'_
>>) -> String
{
588 // For vector registers LLVM wants the register name to match the type size.
589 InlineAsmRegOrRegClass
::Reg(reg
) => {
590 if let Some(idx
) = xmm_reg_index(reg
) {
591 let class
= if let Some(layout
) = layout
{
592 match layout
.size
.bytes() {
598 // We use f32 as the type for discarded outputs
601 format
!("{{{}mm{}}}", class
, idx
)
602 } else if let Some(idx
) = a64_vreg_index(reg
) {
603 let class
= if let Some(layout
) = layout
{
604 match layout
.size
.bytes() {
609 1 => 'd'
, // We fixup i8 to i8x8
613 // We use i64x2 as the type for discarded outputs
616 format
!("{{{}{}}}", class
, idx
)
617 } else if reg
== InlineAsmReg
::AArch64(AArch64InlineAsmReg
::x30
) {
618 // LLVM doesn't recognize x30
620 } else if reg
== InlineAsmReg
::Arm(ArmInlineAsmReg
::r14
) {
621 // LLVM doesn't recognize r14
624 format
!("{{{}}}", reg
.name())
627 InlineAsmRegOrRegClass
::RegClass(reg
) => match reg
{
628 InlineAsmRegClass
::AArch64(AArch64InlineAsmRegClass
::reg
) => "r",
629 InlineAsmRegClass
::AArch64(AArch64InlineAsmRegClass
::vreg
) => "w",
630 InlineAsmRegClass
::AArch64(AArch64InlineAsmRegClass
::vreg_low16
) => "x",
631 InlineAsmRegClass
::AArch64(AArch64InlineAsmRegClass
::preg
) => {
632 unreachable
!("clobber-only")
634 InlineAsmRegClass
::Arm(ArmInlineAsmRegClass
::reg
) => "r",
635 InlineAsmRegClass
::Arm(ArmInlineAsmRegClass
::sreg
)
636 | InlineAsmRegClass
::Arm(ArmInlineAsmRegClass
::dreg_low16
)
637 | InlineAsmRegClass
::Arm(ArmInlineAsmRegClass
::qreg_low8
) => "t",
638 InlineAsmRegClass
::Arm(ArmInlineAsmRegClass
::sreg_low16
)
639 | InlineAsmRegClass
::Arm(ArmInlineAsmRegClass
::dreg_low8
)
640 | InlineAsmRegClass
::Arm(ArmInlineAsmRegClass
::qreg_low4
) => "x",
641 InlineAsmRegClass
::Arm(ArmInlineAsmRegClass
::dreg
)
642 | InlineAsmRegClass
::Arm(ArmInlineAsmRegClass
::qreg
) => "w",
643 InlineAsmRegClass
::Hexagon(HexagonInlineAsmRegClass
::reg
) => "r",
644 InlineAsmRegClass
::Mips(MipsInlineAsmRegClass
::reg
) => "r",
645 InlineAsmRegClass
::Mips(MipsInlineAsmRegClass
::freg
) => "f",
646 InlineAsmRegClass
::Nvptx(NvptxInlineAsmRegClass
::reg16
) => "h",
647 InlineAsmRegClass
::Nvptx(NvptxInlineAsmRegClass
::reg32
) => "r",
648 InlineAsmRegClass
::Nvptx(NvptxInlineAsmRegClass
::reg64
) => "l",
649 InlineAsmRegClass
::PowerPC(PowerPCInlineAsmRegClass
::reg
) => "r",
650 InlineAsmRegClass
::PowerPC(PowerPCInlineAsmRegClass
::reg_nonzero
) => "b",
651 InlineAsmRegClass
::PowerPC(PowerPCInlineAsmRegClass
::freg
) => "f",
652 InlineAsmRegClass
::PowerPC(PowerPCInlineAsmRegClass
::cr
)
653 | InlineAsmRegClass
::PowerPC(PowerPCInlineAsmRegClass
::xer
) => {
654 unreachable
!("clobber-only")
656 InlineAsmRegClass
::RiscV(RiscVInlineAsmRegClass
::reg
) => "r",
657 InlineAsmRegClass
::RiscV(RiscVInlineAsmRegClass
::freg
) => "f",
658 InlineAsmRegClass
::RiscV(RiscVInlineAsmRegClass
::vreg
) => {
659 unreachable
!("clobber-only")
661 InlineAsmRegClass
::X86(X86InlineAsmRegClass
::reg
) => "r",
662 InlineAsmRegClass
::X86(X86InlineAsmRegClass
::reg_abcd
) => "Q",
663 InlineAsmRegClass
::X86(X86InlineAsmRegClass
::reg_byte
) => "q",
664 InlineAsmRegClass
::X86(X86InlineAsmRegClass
::xmm_reg
)
665 | InlineAsmRegClass
::X86(X86InlineAsmRegClass
::ymm_reg
) => "x",
666 InlineAsmRegClass
::X86(X86InlineAsmRegClass
::zmm_reg
) => "v",
667 InlineAsmRegClass
::X86(X86InlineAsmRegClass
::kreg
) => "^Yk",
668 InlineAsmRegClass
::X86(
669 X86InlineAsmRegClass
::x87_reg
| X86InlineAsmRegClass
::mmx_reg
,
670 ) => unreachable
!("clobber-only"),
671 InlineAsmRegClass
::Wasm(WasmInlineAsmRegClass
::local
) => "r",
672 InlineAsmRegClass
::Bpf(BpfInlineAsmRegClass
::reg
) => "r",
673 InlineAsmRegClass
::Bpf(BpfInlineAsmRegClass
::wreg
) => "w",
674 InlineAsmRegClass
::Avr(AvrInlineAsmRegClass
::reg
) => "r",
675 InlineAsmRegClass
::Avr(AvrInlineAsmRegClass
::reg_upper
) => "d",
676 InlineAsmRegClass
::Avr(AvrInlineAsmRegClass
::reg_pair
) => "r",
677 InlineAsmRegClass
::Avr(AvrInlineAsmRegClass
::reg_iw
) => "w",
678 InlineAsmRegClass
::Avr(AvrInlineAsmRegClass
::reg_ptr
) => "e",
679 InlineAsmRegClass
::S390x(S390xInlineAsmRegClass
::reg
) => "r",
680 InlineAsmRegClass
::S390x(S390xInlineAsmRegClass
::freg
) => "f",
681 InlineAsmRegClass
::SpirV(SpirVInlineAsmRegClass
::reg
) => {
682 bug
!("LLVM backend does not support SPIR-V")
684 InlineAsmRegClass
::Err
=> unreachable
!(),
690 /// Converts a modifier into LLVM's equivalent modifier.
693 reg
: InlineAsmRegClass
,
694 modifier
: Option
<char>,
697 InlineAsmRegClass
::AArch64(AArch64InlineAsmRegClass
::reg
) => modifier
,
698 InlineAsmRegClass
::AArch64(AArch64InlineAsmRegClass
::vreg
)
699 | InlineAsmRegClass
::AArch64(AArch64InlineAsmRegClass
::vreg_low16
) => {
700 if modifier
== Some('v'
) { None }
else { modifier }
702 InlineAsmRegClass
::AArch64(AArch64InlineAsmRegClass
::preg
) => {
703 unreachable
!("clobber-only")
705 InlineAsmRegClass
::Arm(ArmInlineAsmRegClass
::reg
) => None
,
706 InlineAsmRegClass
::Arm(ArmInlineAsmRegClass
::sreg
)
707 | InlineAsmRegClass
::Arm(ArmInlineAsmRegClass
::sreg_low16
) => None
,
708 InlineAsmRegClass
::Arm(ArmInlineAsmRegClass
::dreg
)
709 | InlineAsmRegClass
::Arm(ArmInlineAsmRegClass
::dreg_low16
)
710 | InlineAsmRegClass
::Arm(ArmInlineAsmRegClass
::dreg_low8
) => Some('P'
),
711 InlineAsmRegClass
::Arm(ArmInlineAsmRegClass
::qreg
)
712 | InlineAsmRegClass
::Arm(ArmInlineAsmRegClass
::qreg_low8
)
713 | InlineAsmRegClass
::Arm(ArmInlineAsmRegClass
::qreg_low4
) => {
714 if modifier
.is_none() {
720 InlineAsmRegClass
::Hexagon(_
) => None
,
721 InlineAsmRegClass
::Mips(_
) => None
,
722 InlineAsmRegClass
::Nvptx(_
) => None
,
723 InlineAsmRegClass
::PowerPC(_
) => None
,
724 InlineAsmRegClass
::RiscV(RiscVInlineAsmRegClass
::reg
)
725 | InlineAsmRegClass
::RiscV(RiscVInlineAsmRegClass
::freg
) => None
,
726 InlineAsmRegClass
::RiscV(RiscVInlineAsmRegClass
::vreg
) => {
727 unreachable
!("clobber-only")
729 InlineAsmRegClass
::X86(X86InlineAsmRegClass
::reg
)
730 | InlineAsmRegClass
::X86(X86InlineAsmRegClass
::reg_abcd
) => match modifier
{
731 None
if arch
== InlineAsmArch
::X86_64
=> Some('q'
),
733 Some('l'
) => Some('b'
),
734 Some('h'
) => Some('h'
),
735 Some('x'
) => Some('w'
),
736 Some('e'
) => Some('k'
),
737 Some('r'
) => Some('q'
),
740 InlineAsmRegClass
::X86(X86InlineAsmRegClass
::reg_byte
) => None
,
741 InlineAsmRegClass
::X86(reg @ X86InlineAsmRegClass
::xmm_reg
)
742 | InlineAsmRegClass
::X86(reg @ X86InlineAsmRegClass
::ymm_reg
)
743 | InlineAsmRegClass
::X86(reg @ X86InlineAsmRegClass
::zmm_reg
) => match (reg
, modifier
) {
744 (X86InlineAsmRegClass
::xmm_reg
, None
) => Some('x'
),
745 (X86InlineAsmRegClass
::ymm_reg
, None
) => Some('t'
),
746 (X86InlineAsmRegClass
::zmm_reg
, None
) => Some('g'
),
747 (_
, Some('x'
)) => Some('x'
),
748 (_
, Some('y'
)) => Some('t'
),
749 (_
, Some('z'
)) => Some('g'
),
752 InlineAsmRegClass
::X86(X86InlineAsmRegClass
::kreg
) => None
,
753 InlineAsmRegClass
::X86(X86InlineAsmRegClass
::x87_reg
| X86InlineAsmRegClass
::mmx_reg
) => {
754 unreachable
!("clobber-only")
756 InlineAsmRegClass
::Wasm(WasmInlineAsmRegClass
::local
) => None
,
757 InlineAsmRegClass
::Bpf(_
) => None
,
758 InlineAsmRegClass
::Avr(AvrInlineAsmRegClass
::reg_pair
)
759 | InlineAsmRegClass
::Avr(AvrInlineAsmRegClass
::reg_iw
)
760 | InlineAsmRegClass
::Avr(AvrInlineAsmRegClass
::reg_ptr
) => match modifier
{
761 Some('h'
) => Some('B'
),
762 Some('l'
) => Some('A'
),
765 InlineAsmRegClass
::Avr(_
) => None
,
766 InlineAsmRegClass
::S390x(_
) => None
,
767 InlineAsmRegClass
::SpirV(SpirVInlineAsmRegClass
::reg
) => {
768 bug
!("LLVM backend does not support SPIR-V")
770 InlineAsmRegClass
::Err
=> unreachable
!(),
774 /// Type to use for outputs that are discarded. It doesn't really matter what
775 /// the type is, as long as it is valid for the constraint code.
776 fn dummy_output_type
<'ll
>(cx
: &CodegenCx
<'ll
, '_
>, reg
: InlineAsmRegClass
) -> &'ll Type
{
778 InlineAsmRegClass
::AArch64(AArch64InlineAsmRegClass
::reg
) => cx
.type_i32(),
779 InlineAsmRegClass
::AArch64(AArch64InlineAsmRegClass
::vreg
)
780 | InlineAsmRegClass
::AArch64(AArch64InlineAsmRegClass
::vreg_low16
) => {
781 cx
.type_vector(cx
.type_i64(), 2)
783 InlineAsmRegClass
::AArch64(AArch64InlineAsmRegClass
::preg
) => {
784 unreachable
!("clobber-only")
786 InlineAsmRegClass
::Arm(ArmInlineAsmRegClass
::reg
) => cx
.type_i32(),
787 InlineAsmRegClass
::Arm(ArmInlineAsmRegClass
::sreg
)
788 | InlineAsmRegClass
::Arm(ArmInlineAsmRegClass
::sreg_low16
) => cx
.type_f32(),
789 InlineAsmRegClass
::Arm(ArmInlineAsmRegClass
::dreg
)
790 | InlineAsmRegClass
::Arm(ArmInlineAsmRegClass
::dreg_low16
)
791 | InlineAsmRegClass
::Arm(ArmInlineAsmRegClass
::dreg_low8
) => cx
.type_f64(),
792 InlineAsmRegClass
::Arm(ArmInlineAsmRegClass
::qreg
)
793 | InlineAsmRegClass
::Arm(ArmInlineAsmRegClass
::qreg_low8
)
794 | InlineAsmRegClass
::Arm(ArmInlineAsmRegClass
::qreg_low4
) => {
795 cx
.type_vector(cx
.type_i64(), 2)
797 InlineAsmRegClass
::Hexagon(HexagonInlineAsmRegClass
::reg
) => cx
.type_i32(),
798 InlineAsmRegClass
::Mips(MipsInlineAsmRegClass
::reg
) => cx
.type_i32(),
799 InlineAsmRegClass
::Mips(MipsInlineAsmRegClass
::freg
) => cx
.type_f32(),
800 InlineAsmRegClass
::Nvptx(NvptxInlineAsmRegClass
::reg16
) => cx
.type_i16(),
801 InlineAsmRegClass
::Nvptx(NvptxInlineAsmRegClass
::reg32
) => cx
.type_i32(),
802 InlineAsmRegClass
::Nvptx(NvptxInlineAsmRegClass
::reg64
) => cx
.type_i64(),
803 InlineAsmRegClass
::PowerPC(PowerPCInlineAsmRegClass
::reg
) => cx
.type_i32(),
804 InlineAsmRegClass
::PowerPC(PowerPCInlineAsmRegClass
::reg_nonzero
) => cx
.type_i32(),
805 InlineAsmRegClass
::PowerPC(PowerPCInlineAsmRegClass
::freg
) => cx
.type_f64(),
806 InlineAsmRegClass
::PowerPC(PowerPCInlineAsmRegClass
::cr
)
807 | InlineAsmRegClass
::PowerPC(PowerPCInlineAsmRegClass
::xer
) => {
808 unreachable
!("clobber-only")
810 InlineAsmRegClass
::RiscV(RiscVInlineAsmRegClass
::reg
) => cx
.type_i32(),
811 InlineAsmRegClass
::RiscV(RiscVInlineAsmRegClass
::freg
) => cx
.type_f32(),
812 InlineAsmRegClass
::RiscV(RiscVInlineAsmRegClass
::vreg
) => {
813 unreachable
!("clobber-only")
815 InlineAsmRegClass
::X86(X86InlineAsmRegClass
::reg
)
816 | InlineAsmRegClass
::X86(X86InlineAsmRegClass
::reg_abcd
) => cx
.type_i32(),
817 InlineAsmRegClass
::X86(X86InlineAsmRegClass
::reg_byte
) => cx
.type_i8(),
818 InlineAsmRegClass
::X86(X86InlineAsmRegClass
::xmm_reg
)
819 | InlineAsmRegClass
::X86(X86InlineAsmRegClass
::ymm_reg
)
820 | InlineAsmRegClass
::X86(X86InlineAsmRegClass
::zmm_reg
) => cx
.type_f32(),
821 InlineAsmRegClass
::X86(X86InlineAsmRegClass
::kreg
) => cx
.type_i16(),
822 InlineAsmRegClass
::X86(X86InlineAsmRegClass
::x87_reg
| X86InlineAsmRegClass
::mmx_reg
) => {
823 unreachable
!("clobber-only")
825 InlineAsmRegClass
::Wasm(WasmInlineAsmRegClass
::local
) => cx
.type_i32(),
826 InlineAsmRegClass
::Bpf(BpfInlineAsmRegClass
::reg
) => cx
.type_i64(),
827 InlineAsmRegClass
::Bpf(BpfInlineAsmRegClass
::wreg
) => cx
.type_i32(),
828 InlineAsmRegClass
::Avr(AvrInlineAsmRegClass
::reg
) => cx
.type_i8(),
829 InlineAsmRegClass
::Avr(AvrInlineAsmRegClass
::reg_upper
) => cx
.type_i8(),
830 InlineAsmRegClass
::Avr(AvrInlineAsmRegClass
::reg_pair
) => cx
.type_i16(),
831 InlineAsmRegClass
::Avr(AvrInlineAsmRegClass
::reg_iw
) => cx
.type_i16(),
832 InlineAsmRegClass
::Avr(AvrInlineAsmRegClass
::reg_ptr
) => cx
.type_i16(),
833 InlineAsmRegClass
::S390x(S390xInlineAsmRegClass
::reg
) => cx
.type_i32(),
834 InlineAsmRegClass
::S390x(S390xInlineAsmRegClass
::freg
) => cx
.type_f64(),
835 InlineAsmRegClass
::SpirV(SpirVInlineAsmRegClass
::reg
) => {
836 bug
!("LLVM backend does not support SPIR-V")
838 InlineAsmRegClass
::Err
=> unreachable
!(),
842 /// Helper function to get the LLVM type for a Scalar. Pointers are returned as
843 /// the equivalent integer type.
844 fn llvm_asm_scalar_type
<'ll
>(cx
: &CodegenCx
<'ll
, '_
>, scalar
: Scalar
) -> &'ll Type
{
846 Primitive
::Int(Integer
::I8
, _
) => cx
.type_i8(),
847 Primitive
::Int(Integer
::I16
, _
) => cx
.type_i16(),
848 Primitive
::Int(Integer
::I32
, _
) => cx
.type_i32(),
849 Primitive
::Int(Integer
::I64
, _
) => cx
.type_i64(),
850 Primitive
::F32
=> cx
.type_f32(),
851 Primitive
::F64
=> cx
.type_f64(),
852 Primitive
::Pointer
=> cx
.type_isize(),
857 /// Fix up an input value to work around LLVM bugs.
858 fn llvm_fixup_input
<'ll
, 'tcx
>(
859 bx
: &mut Builder
<'_
, 'll
, 'tcx
>,
860 mut value
: &'ll Value
,
861 reg
: InlineAsmRegClass
,
862 layout
: &TyAndLayout
<'tcx
>,
864 match (reg
, layout
.abi
) {
865 (InlineAsmRegClass
::AArch64(AArch64InlineAsmRegClass
::vreg
), Abi
::Scalar(s
)) => {
866 if let Primitive
::Int(Integer
::I8
, _
) = s
.value
{
867 let vec_ty
= bx
.cx
.type_vector(bx
.cx
.type_i8(), 8);
868 bx
.insert_element(bx
.const_undef(vec_ty
), value
, bx
.const_i32(0))
873 (InlineAsmRegClass
::AArch64(AArch64InlineAsmRegClass
::vreg_low16
), Abi
::Scalar(s
)) => {
874 let elem_ty
= llvm_asm_scalar_type(bx
.cx
, s
);
875 let count
= 16 / layout
.size
.bytes();
876 let vec_ty
= bx
.cx
.type_vector(elem_ty
, count
);
877 if let Primitive
::Pointer
= s
.value
{
878 value
= bx
.ptrtoint(value
, bx
.cx
.type_isize());
880 bx
.insert_element(bx
.const_undef(vec_ty
), value
, bx
.const_i32(0))
883 InlineAsmRegClass
::AArch64(AArch64InlineAsmRegClass
::vreg_low16
),
884 Abi
::Vector { element, count }
,
885 ) if layout
.size
.bytes() == 8 => {
886 let elem_ty
= llvm_asm_scalar_type(bx
.cx
, element
);
887 let vec_ty
= bx
.cx
.type_vector(elem_ty
, count
);
888 let indices
: Vec
<_
> = (0..count
* 2).map(|x
| bx
.const_i32(x
as i32)).collect();
889 bx
.shuffle_vector(value
, bx
.const_undef(vec_ty
), bx
.const_vector(&indices
))
891 (InlineAsmRegClass
::X86(X86InlineAsmRegClass
::reg_abcd
), Abi
::Scalar(s
))
892 if s
.value
== Primitive
::F64
=>
894 bx
.bitcast(value
, bx
.cx
.type_i64())
897 InlineAsmRegClass
::X86(X86InlineAsmRegClass
::xmm_reg
| X86InlineAsmRegClass
::zmm_reg
),
899 ) if layout
.size
.bytes() == 64 => bx
.bitcast(value
, bx
.cx
.type_vector(bx
.cx
.type_f64(), 8)),
901 InlineAsmRegClass
::Arm(ArmInlineAsmRegClass
::sreg
| ArmInlineAsmRegClass
::sreg_low16
),
904 if let Primitive
::Int(Integer
::I32
, _
) = s
.value
{
905 bx
.bitcast(value
, bx
.cx
.type_f32())
911 InlineAsmRegClass
::Arm(
912 ArmInlineAsmRegClass
::dreg
913 | ArmInlineAsmRegClass
::dreg_low8
914 | ArmInlineAsmRegClass
::dreg_low16
,
918 if let Primitive
::Int(Integer
::I64
, _
) = s
.value
{
919 bx
.bitcast(value
, bx
.cx
.type_f64())
924 (InlineAsmRegClass
::Mips(MipsInlineAsmRegClass
::reg
), Abi
::Scalar(s
)) => match s
.value
{
925 // MIPS only supports register-length arithmetics.
926 Primitive
::Int(Integer
::I8
| Integer
::I16
, _
) => bx
.zext(value
, bx
.cx
.type_i32()),
927 Primitive
::F32
=> bx
.bitcast(value
, bx
.cx
.type_i32()),
928 Primitive
::F64
=> bx
.bitcast(value
, bx
.cx
.type_i64()),
935 /// Fix up an output value to work around LLVM bugs.
936 fn llvm_fixup_output
<'ll
, 'tcx
>(
937 bx
: &mut Builder
<'_
, 'll
, 'tcx
>,
938 mut value
: &'ll Value
,
939 reg
: InlineAsmRegClass
,
940 layout
: &TyAndLayout
<'tcx
>,
942 match (reg
, layout
.abi
) {
943 (InlineAsmRegClass
::AArch64(AArch64InlineAsmRegClass
::vreg
), Abi
::Scalar(s
)) => {
944 if let Primitive
::Int(Integer
::I8
, _
) = s
.value
{
945 bx
.extract_element(value
, bx
.const_i32(0))
950 (InlineAsmRegClass
::AArch64(AArch64InlineAsmRegClass
::vreg_low16
), Abi
::Scalar(s
)) => {
951 value
= bx
.extract_element(value
, bx
.const_i32(0));
952 if let Primitive
::Pointer
= s
.value
{
953 value
= bx
.inttoptr(value
, layout
.llvm_type(bx
.cx
));
958 InlineAsmRegClass
::AArch64(AArch64InlineAsmRegClass
::vreg_low16
),
959 Abi
::Vector { element, count }
,
960 ) if layout
.size
.bytes() == 8 => {
961 let elem_ty
= llvm_asm_scalar_type(bx
.cx
, element
);
962 let vec_ty
= bx
.cx
.type_vector(elem_ty
, count
* 2);
963 let indices
: Vec
<_
> = (0..count
).map(|x
| bx
.const_i32(x
as i32)).collect();
964 bx
.shuffle_vector(value
, bx
.const_undef(vec_ty
), bx
.const_vector(&indices
))
966 (InlineAsmRegClass
::X86(X86InlineAsmRegClass
::reg_abcd
), Abi
::Scalar(s
))
967 if s
.value
== Primitive
::F64
=>
969 bx
.bitcast(value
, bx
.cx
.type_f64())
972 InlineAsmRegClass
::X86(X86InlineAsmRegClass
::xmm_reg
| X86InlineAsmRegClass
::zmm_reg
),
974 ) if layout
.size
.bytes() == 64 => bx
.bitcast(value
, layout
.llvm_type(bx
.cx
)),
976 InlineAsmRegClass
::Arm(ArmInlineAsmRegClass
::sreg
| ArmInlineAsmRegClass
::sreg_low16
),
979 if let Primitive
::Int(Integer
::I32
, _
) = s
.value
{
980 bx
.bitcast(value
, bx
.cx
.type_i32())
986 InlineAsmRegClass
::Arm(
987 ArmInlineAsmRegClass
::dreg
988 | ArmInlineAsmRegClass
::dreg_low8
989 | ArmInlineAsmRegClass
::dreg_low16
,
993 if let Primitive
::Int(Integer
::I64
, _
) = s
.value
{
994 bx
.bitcast(value
, bx
.cx
.type_i64())
999 (InlineAsmRegClass
::Mips(MipsInlineAsmRegClass
::reg
), Abi
::Scalar(s
)) => match s
.value
{
1000 // MIPS only supports register-length arithmetics.
1001 Primitive
::Int(Integer
::I8
, _
) => bx
.trunc(value
, bx
.cx
.type_i8()),
1002 Primitive
::Int(Integer
::I16
, _
) => bx
.trunc(value
, bx
.cx
.type_i16()),
1003 Primitive
::F32
=> bx
.bitcast(value
, bx
.cx
.type_f32()),
1004 Primitive
::F64
=> bx
.bitcast(value
, bx
.cx
.type_f64()),
1011 /// Output type to use for llvm_fixup_output.
1012 fn llvm_fixup_output_type
<'ll
, 'tcx
>(
1013 cx
: &CodegenCx
<'ll
, 'tcx
>,
1014 reg
: InlineAsmRegClass
,
1015 layout
: &TyAndLayout
<'tcx
>,
1017 match (reg
, layout
.abi
) {
1018 (InlineAsmRegClass
::AArch64(AArch64InlineAsmRegClass
::vreg
), Abi
::Scalar(s
)) => {
1019 if let Primitive
::Int(Integer
::I8
, _
) = s
.value
{
1020 cx
.type_vector(cx
.type_i8(), 8)
1022 layout
.llvm_type(cx
)
1025 (InlineAsmRegClass
::AArch64(AArch64InlineAsmRegClass
::vreg_low16
), Abi
::Scalar(s
)) => {
1026 let elem_ty
= llvm_asm_scalar_type(cx
, s
);
1027 let count
= 16 / layout
.size
.bytes();
1028 cx
.type_vector(elem_ty
, count
)
1031 InlineAsmRegClass
::AArch64(AArch64InlineAsmRegClass
::vreg_low16
),
1032 Abi
::Vector { element, count }
,
1033 ) if layout
.size
.bytes() == 8 => {
1034 let elem_ty
= llvm_asm_scalar_type(cx
, element
);
1035 cx
.type_vector(elem_ty
, count
* 2)
1037 (InlineAsmRegClass
::X86(X86InlineAsmRegClass
::reg_abcd
), Abi
::Scalar(s
))
1038 if s
.value
== Primitive
::F64
=>
1043 InlineAsmRegClass
::X86(X86InlineAsmRegClass
::xmm_reg
| X86InlineAsmRegClass
::zmm_reg
),
1045 ) if layout
.size
.bytes() == 64 => cx
.type_vector(cx
.type_f64(), 8),
1047 InlineAsmRegClass
::Arm(ArmInlineAsmRegClass
::sreg
| ArmInlineAsmRegClass
::sreg_low16
),
1050 if let Primitive
::Int(Integer
::I32
, _
) = s
.value
{
1053 layout
.llvm_type(cx
)
1057 InlineAsmRegClass
::Arm(
1058 ArmInlineAsmRegClass
::dreg
1059 | ArmInlineAsmRegClass
::dreg_low8
1060 | ArmInlineAsmRegClass
::dreg_low16
,
1064 if let Primitive
::Int(Integer
::I64
, _
) = s
.value
{
1067 layout
.llvm_type(cx
)
1070 (InlineAsmRegClass
::Mips(MipsInlineAsmRegClass
::reg
), Abi
::Scalar(s
)) => match s
.value
{
1071 // MIPS only supports register-length arithmetics.
1072 Primitive
::Int(Integer
::I8
| Integer
::I16
, _
) => cx
.type_i32(),
1073 Primitive
::F32
=> cx
.type_i32(),
1074 Primitive
::F64
=> cx
.type_i64(),
1075 _
=> layout
.llvm_type(cx
),
1077 _
=> layout
.llvm_type(cx
),