]>
Commit | Line | Data |
---|---|---|
5e7ed085 | 1 | use crate::attributes; |
dfeec247 | 2 | use crate::builder::Builder; |
a2a8927a | 3 | use crate::common::Funclet; |
9fa01778 | 4 | use crate::context::CodegenCx; |
dfeec247 | 5 | use crate::llvm; |
f9f354fc | 6 | use crate::type_::Type; |
9fa01778 | 7 | use crate::type_of::LayoutLlvmExt; |
9fa01778 | 8 | use crate::value::Value; |
54a0048b | 9 | |
3dfed10e | 10 | use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece}; |
a1dfa0c6 | 11 | use rustc_codegen_ssa::mir::operand::OperandValue; |
dfeec247 | 12 | use rustc_codegen_ssa::traits::*; |
f9f354fc | 13 | use rustc_data_structures::fx::FxHashMap; |
f9f354fc | 14 | use rustc_middle::ty::layout::TyAndLayout; |
3c0e092e | 15 | use rustc_middle::{bug, span_bug, ty::Instance}; |
5099ac24 | 16 | use rustc_span::{Pos, Span}; |
f9f354fc XL |
17 | use rustc_target::abi::*; |
18 | use rustc_target::asm::*; | |
32a655c1 | 19 | |
dfeec247 | 20 | use libc::{c_char, c_uint}; |
5e7ed085 | 21 | use smallvec::SmallVec; |
a1dfa0c6 | 22 | |
a2a8927a | 23 | impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { |
f9f354fc XL |
24 | fn codegen_inline_asm( |
25 | &mut self, | |
26 | template: &[InlineAsmTemplatePiece], | |
27 | operands: &[InlineAsmOperandRef<'tcx, Self>], | |
28 | options: InlineAsmOptions, | |
29 | line_spans: &[Span], | |
3c0e092e | 30 | instance: Instance<'_>, |
a2a8927a | 31 | dest_catch_funclet: Option<(Self::BasicBlock, Self::BasicBlock, Option<&Self::Funclet>)>, |
f9f354fc XL |
32 | ) { |
33 | let asm_arch = self.tcx.sess.asm_arch.unwrap(); | |
54a0048b | 34 | |
f9f354fc XL |
35 | // Collect the types of output operands |
36 | let mut constraints = vec![]; | |
cdc7bbd5 | 37 | let mut clobbers = vec![]; |
f9f354fc XL |
38 | let mut output_types = vec![]; |
39 | let mut op_idx = FxHashMap::default(); | |
136023e0 | 40 | let mut clobbered_x87 = false; |
f9f354fc XL |
41 | for (idx, op) in operands.iter().enumerate() { |
42 | match *op { | |
43 | InlineAsmOperandRef::Out { reg, late, place } => { | |
cdc7bbd5 XL |
44 | let is_target_supported = |reg_class: InlineAsmRegClass| { |
45 | for &(_, feature) in reg_class.supported_types(asm_arch) { | |
46 | if let Some(feature) = feature { | |
3c0e092e | 47 | let codegen_fn_attrs = self.tcx.codegen_fn_attrs(instance.def_id()); |
5099ac24 FG |
48 | if self.tcx.sess.target_features.contains(&feature) |
49 | || codegen_fn_attrs.target_features.contains(&feature) | |
cdc7bbd5 XL |
50 | { |
51 | return true; | |
52 | } | |
53 | } else { | |
54 | // Register class is unconditionally supported | |
55 | return true; | |
56 | } | |
57 | } | |
58 | false | |
59 | }; | |
60 | ||
3dfed10e XL |
61 | let mut layout = None; |
62 | let ty = if let Some(ref place) = place { | |
63 | layout = Some(&place.layout); | |
f9f354fc | 64 | llvm_fixup_output_type(self.cx, reg.reg_class(), &place.layout) |
136023e0 XL |
65 | } else if matches!( |
66 | reg.reg_class(), | |
67 | InlineAsmRegClass::X86( | |
68 | X86InlineAsmRegClass::mmx_reg | X86InlineAsmRegClass::x87_reg | |
69 | ) | |
70 | ) { | |
71 | // Special handling for x87/mmx registers: we always | |
72 | // clobber the whole set if one register is marked as | |
73 | // clobbered. This is due to the way LLVM handles the | |
74 | // FP stack in inline assembly. | |
75 | if !clobbered_x87 { | |
76 | clobbered_x87 = true; | |
77 | clobbers.push("~{st}".to_string()); | |
78 | for i in 1..=7 { | |
79 | clobbers.push(format!("~{{st({})}}", i)); | |
80 | } | |
81 | } | |
82 | continue; | |
83 | } else if !is_target_supported(reg.reg_class()) | |
84 | || reg.reg_class().is_clobber_only(asm_arch) | |
85 | { | |
cdc7bbd5 XL |
86 | // We turn discarded outputs into clobber constraints |
87 | // if the target feature needed by the register class is | |
88 | // disabled. This is necessary otherwise LLVM will try | |
89 | // to actually allocate a register for the dummy output. | |
90 | assert!(matches!(reg, InlineAsmRegOrRegClass::Reg(_))); | |
91 | clobbers.push(format!("~{}", reg_to_llvm(reg, None))); | |
92 | continue; | |
f9f354fc XL |
93 | } else { |
94 | // If the output is discarded, we don't really care what | |
95 | // type is used. We're just using this to tell LLVM to | |
96 | // reserve the register. | |
97 | dummy_output_type(self.cx, reg.reg_class()) | |
98 | }; | |
99 | output_types.push(ty); | |
100 | op_idx.insert(idx, constraints.len()); | |
101 | let prefix = if late { "=" } else { "=&" }; | |
3dfed10e | 102 | constraints.push(format!("{}{}", prefix, reg_to_llvm(reg, layout))); |
f9f354fc XL |
103 | } |
104 | InlineAsmOperandRef::InOut { reg, late, in_value, out_place } => { | |
3dfed10e XL |
105 | let layout = if let Some(ref out_place) = out_place { |
106 | &out_place.layout | |
f9f354fc XL |
107 | } else { |
108 | // LLVM required tied operands to have the same type, | |
109 | // so we just use the type of the input. | |
3dfed10e | 110 | &in_value.layout |
f9f354fc | 111 | }; |
3dfed10e | 112 | let ty = llvm_fixup_output_type(self.cx, reg.reg_class(), layout); |
f9f354fc XL |
113 | output_types.push(ty); |
114 | op_idx.insert(idx, constraints.len()); | |
115 | let prefix = if late { "=" } else { "=&" }; | |
3dfed10e | 116 | constraints.push(format!("{}{}", prefix, reg_to_llvm(reg, Some(layout)))); |
f9f354fc XL |
117 | } |
118 | _ => {} | |
119 | } | |
a1dfa0c6 | 120 | } |
0bf4aa26 | 121 | |
f9f354fc XL |
122 | // Collect input operands |
123 | let mut inputs = vec![]; | |
124 | for (idx, op) in operands.iter().enumerate() { | |
125 | match *op { | |
126 | InlineAsmOperandRef::In { reg, value } => { | |
3dfed10e | 127 | let llval = |
f9f354fc | 128 | llvm_fixup_input(self, value.immediate(), reg.reg_class(), &value.layout); |
3dfed10e | 129 | inputs.push(llval); |
f9f354fc | 130 | op_idx.insert(idx, constraints.len()); |
3dfed10e | 131 | constraints.push(reg_to_llvm(reg, Some(&value.layout))); |
f9f354fc | 132 | } |
487cf647 | 133 | InlineAsmOperandRef::InOut { reg, late, in_value, out_place: _ } => { |
f9f354fc XL |
134 | let value = llvm_fixup_input( |
135 | self, | |
136 | in_value.immediate(), | |
137 | reg.reg_class(), | |
138 | &in_value.layout, | |
139 | ); | |
140 | inputs.push(value); | |
487cf647 FG |
141 | |
142 | // In the case of fixed registers, we have the choice of | |
143 | // either using a tied operand or duplicating the constraint. | |
144 | // We prefer the latter because it matches the behavior of | |
145 | // Clang. | |
146 | if late && matches!(reg, InlineAsmRegOrRegClass::Reg(_)) { | |
9c376795 | 147 | constraints.push(reg_to_llvm(reg, Some(&in_value.layout)).to_string()); |
487cf647 FG |
148 | } else { |
149 | constraints.push(format!("{}", op_idx[&idx])); | |
150 | } | |
f9f354fc XL |
151 | } |
152 | InlineAsmOperandRef::SymFn { instance } => { | |
153 | inputs.push(self.cx.get_fn(instance)); | |
154 | op_idx.insert(idx, constraints.len()); | |
155 | constraints.push("s".to_string()); | |
156 | } | |
157 | InlineAsmOperandRef::SymStatic { def_id } => { | |
158 | inputs.push(self.cx.get_static(def_id)); | |
159 | op_idx.insert(idx, constraints.len()); | |
160 | constraints.push("s".to_string()); | |
161 | } | |
162 | _ => {} | |
163 | } | |
164 | } | |
165 | ||
166 | // Build the template string | |
167 | let mut template_str = String::new(); | |
168 | for piece in template { | |
169 | match *piece { | |
170 | InlineAsmTemplatePiece::String(ref s) => { | |
171 | if s.contains('$') { | |
172 | for c in s.chars() { | |
173 | if c == '$' { | |
174 | template_str.push_str("$$"); | |
175 | } else { | |
176 | template_str.push(c); | |
177 | } | |
178 | } | |
179 | } else { | |
180 | template_str.push_str(s) | |
181 | } | |
182 | } | |
183 | InlineAsmTemplatePiece::Placeholder { operand_idx, modifier, span: _ } => { | |
184 | match operands[operand_idx] { | |
185 | InlineAsmOperandRef::In { reg, .. } | |
186 | | InlineAsmOperandRef::Out { reg, .. } | |
187 | | InlineAsmOperandRef::InOut { reg, .. } => { | |
188 | let modifier = modifier_to_llvm(asm_arch, reg.reg_class(), modifier); | |
189 | if let Some(modifier) = modifier { | |
190 | template_str.push_str(&format!( | |
191 | "${{{}:{}}}", | |
192 | op_idx[&operand_idx], modifier | |
193 | )); | |
194 | } else { | |
195 | template_str.push_str(&format!("${{{}}}", op_idx[&operand_idx])); | |
196 | } | |
197 | } | |
198 | InlineAsmOperandRef::Const { ref string } => { | |
199 | // Const operands get injected directly into the template | |
200 | template_str.push_str(string); | |
201 | } | |
202 | InlineAsmOperandRef::SymFn { .. } | |
203 | | InlineAsmOperandRef::SymStatic { .. } => { | |
204 | // Only emit the raw symbol name | |
205 | template_str.push_str(&format!("${{{}:c}}", op_idx[&operand_idx])); | |
206 | } | |
207 | } | |
208 | } | |
209 | } | |
210 | } | |
211 | ||
cdc7bbd5 | 212 | constraints.append(&mut clobbers); |
f9f354fc XL |
213 | if !options.contains(InlineAsmOptions::PRESERVES_FLAGS) { |
214 | match asm_arch { | |
215 | InlineAsmArch::AArch64 | InlineAsmArch::Arm => { | |
216 | constraints.push("~{cc}".to_string()); | |
217 | } | |
218 | InlineAsmArch::X86 | InlineAsmArch::X86_64 => { | |
219 | constraints.extend_from_slice(&[ | |
220 | "~{dirflag}".to_string(), | |
221 | "~{fpsr}".to_string(), | |
222 | "~{flags}".to_string(), | |
223 | ]); | |
224 | } | |
94222f64 XL |
225 | InlineAsmArch::RiscV32 | InlineAsmArch::RiscV64 => { |
226 | constraints.extend_from_slice(&[ | |
227 | "~{vtype}".to_string(), | |
228 | "~{vl}".to_string(), | |
229 | "~{vxsat}".to_string(), | |
230 | "~{vxrm}".to_string(), | |
231 | ]); | |
232 | } | |
a2a8927a XL |
233 | InlineAsmArch::Avr => { |
234 | constraints.push("~{sreg}".to_string()); | |
235 | } | |
f9f354fc | 236 | InlineAsmArch::Nvptx64 => {} |
17df50a5 | 237 | InlineAsmArch::PowerPC | InlineAsmArch::PowerPC64 => {} |
f035d41b | 238 | InlineAsmArch::Hexagon => {} |
49aad941 FG |
239 | InlineAsmArch::LoongArch64 => { |
240 | constraints.extend_from_slice(&[ | |
241 | "~{$fcc0}".to_string(), | |
242 | "~{$fcc1}".to_string(), | |
243 | "~{$fcc2}".to_string(), | |
244 | "~{$fcc3}".to_string(), | |
245 | "~{$fcc4}".to_string(), | |
246 | "~{$fcc5}".to_string(), | |
247 | "~{$fcc6}".to_string(), | |
248 | "~{$fcc7}".to_string(), | |
249 | ]); | |
250 | } | |
29967ef6 | 251 | InlineAsmArch::Mips | InlineAsmArch::Mips64 => {} |
49aad941 FG |
252 | InlineAsmArch::S390x => { |
253 | constraints.push("~{cc}".to_string()); | |
254 | } | |
29967ef6 | 255 | InlineAsmArch::SpirV => {} |
3c0e092e | 256 | InlineAsmArch::Wasm32 | InlineAsmArch::Wasm64 => {} |
17df50a5 | 257 | InlineAsmArch::Bpf => {} |
5099ac24 FG |
258 | InlineAsmArch::Msp430 => { |
259 | constraints.push("~{sr}".to_string()); | |
260 | } | |
353b0b11 FG |
261 | InlineAsmArch::M68k => { |
262 | constraints.push("~{ccr}".to_string()); | |
263 | } | |
f9f354fc XL |
264 | } |
265 | } | |
266 | if !options.contains(InlineAsmOptions::NOMEM) { | |
267 | // This is actually ignored by LLVM, but it's probably best to keep | |
268 | // it just in case. LLVM instead uses the ReadOnly/ReadNone | |
269 | // attributes on the call instruction to optimize. | |
270 | constraints.push("~{memory}".to_string()); | |
271 | } | |
272 | let volatile = !options.contains(InlineAsmOptions::PURE); | |
273 | let alignstack = !options.contains(InlineAsmOptions::NOSTACK); | |
274 | let output_type = match &output_types[..] { | |
275 | [] => self.type_void(), | |
276 | [ty] => ty, | |
c295e0f8 | 277 | tys => self.type_struct(tys, false), |
f9f354fc XL |
278 | }; |
279 | let dialect = match asm_arch { | |
280 | InlineAsmArch::X86 | InlineAsmArch::X86_64 | |
281 | if !options.contains(InlineAsmOptions::ATT_SYNTAX) => | |
282 | { | |
5099ac24 | 283 | llvm::AsmDialect::Intel |
f9f354fc | 284 | } |
5099ac24 | 285 | _ => llvm::AsmDialect::Att, |
f9f354fc XL |
286 | }; |
287 | let result = inline_asm_call( | |
288 | self, | |
289 | &template_str, | |
290 | &constraints.join(","), | |
291 | &inputs, | |
292 | output_type, | |
293 | volatile, | |
294 | alignstack, | |
295 | dialect, | |
296 | line_spans, | |
a2a8927a XL |
297 | options.contains(InlineAsmOptions::MAY_UNWIND), |
298 | dest_catch_funclet, | |
f9f354fc XL |
299 | ) |
300 | .unwrap_or_else(|| span_bug!(line_spans[0], "LLVM asm constraint validation failed")); | |
301 | ||
5e7ed085 | 302 | let mut attrs = SmallVec::<[_; 2]>::new(); |
f9f354fc XL |
303 | if options.contains(InlineAsmOptions::PURE) { |
304 | if options.contains(InlineAsmOptions::NOMEM) { | |
487cf647 | 305 | attrs.push(llvm::MemoryEffects::None.create_attr(self.cx.llcx)); |
f9f354fc | 306 | } else if options.contains(InlineAsmOptions::READONLY) { |
487cf647 | 307 | attrs.push(llvm::MemoryEffects::ReadOnly.create_attr(self.cx.llcx)); |
f9f354fc | 308 | } |
5e7ed085 | 309 | attrs.push(llvm::AttributeKind::WillReturn.create_attr(self.cx.llcx)); |
29967ef6 | 310 | } else if options.contains(InlineAsmOptions::NOMEM) { |
487cf647 | 311 | attrs.push(llvm::MemoryEffects::InaccessibleMemOnly.create_attr(self.cx.llcx)); |
f9f354fc | 312 | } else { |
29967ef6 | 313 | // LLVM doesn't have an attribute to represent ReadOnly + SideEffect |
f9f354fc | 314 | } |
5e7ed085 | 315 | attributes::apply_to_callsite(result, llvm::AttributePlace::Function, &{ attrs }); |
f9f354fc | 316 | |
04454e1e FG |
317 | // Switch to the 'normal' basic block if we did an `invoke` instead of a `call` |
318 | if let Some((dest, _, _)) = dest_catch_funclet { | |
319 | self.switch_to_block(dest); | |
320 | } | |
321 | ||
f9f354fc XL |
322 | // Write results to outputs |
323 | for (idx, op) in operands.iter().enumerate() { | |
324 | if let InlineAsmOperandRef::Out { reg, place: Some(place), .. } | |
325 | | InlineAsmOperandRef::InOut { reg, out_place: Some(place), .. } = *op | |
326 | { | |
327 | let value = if output_types.len() == 1 { | |
328 | result | |
329 | } else { | |
330 | self.extract_value(result, op_idx[&idx] as u64) | |
331 | }; | |
332 | let value = llvm_fixup_output(self, value, reg.reg_class(), &place.layout); | |
333 | OperandValue::Immediate(value).store(self, place); | |
334 | } | |
335 | } | |
a1dfa0c6 | 336 | } |
54a0048b | 337 | } |
cc61c64b | 338 | |
04454e1e | 339 | impl<'tcx> AsmMethods<'tcx> for CodegenCx<'_, 'tcx> { |
17df50a5 XL |
340 | fn codegen_global_asm( |
341 | &self, | |
342 | template: &[InlineAsmTemplatePiece], | |
04454e1e | 343 | operands: &[GlobalAsmOperandRef<'tcx>], |
17df50a5 XL |
344 | options: InlineAsmOptions, |
345 | _line_spans: &[Span], | |
346 | ) { | |
347 | let asm_arch = self.tcx.sess.asm_arch.unwrap(); | |
348 | ||
349 | // Default to Intel syntax on x86 | |
350 | let intel_syntax = matches!(asm_arch, InlineAsmArch::X86 | InlineAsmArch::X86_64) | |
351 | && !options.contains(InlineAsmOptions::ATT_SYNTAX); | |
352 | ||
353 | // Build the template string | |
354 | let mut template_str = String::new(); | |
355 | if intel_syntax { | |
356 | template_str.push_str(".intel_syntax\n"); | |
357 | } | |
358 | for piece in template { | |
359 | match *piece { | |
360 | InlineAsmTemplatePiece::String(ref s) => template_str.push_str(s), | |
361 | InlineAsmTemplatePiece::Placeholder { operand_idx, modifier: _, span: _ } => { | |
362 | match operands[operand_idx] { | |
363 | GlobalAsmOperandRef::Const { ref string } => { | |
364 | // Const operands get injected directly into the | |
365 | // template. Note that we don't need to escape $ | |
366 | // here unlike normal inline assembly. | |
367 | template_str.push_str(string); | |
368 | } | |
04454e1e FG |
369 | GlobalAsmOperandRef::SymFn { instance } => { |
370 | let llval = self.get_fn(instance); | |
371 | self.add_compiler_used_global(llval); | |
372 | let symbol = llvm::build_string(|s| unsafe { | |
373 | llvm::LLVMRustGetMangledName(llval, s); | |
374 | }) | |
375 | .expect("symbol is not valid UTF-8"); | |
376 | template_str.push_str(&symbol); | |
377 | } | |
378 | GlobalAsmOperandRef::SymStatic { def_id } => { | |
379 | let llval = self | |
380 | .renamed_statics | |
381 | .borrow() | |
382 | .get(&def_id) | |
383 | .copied() | |
384 | .unwrap_or_else(|| self.get_static(def_id)); | |
385 | self.add_compiler_used_global(llval); | |
386 | let symbol = llvm::build_string(|s| unsafe { | |
387 | llvm::LLVMRustGetMangledName(llval, s); | |
388 | }) | |
389 | .expect("symbol is not valid UTF-8"); | |
390 | template_str.push_str(&symbol); | |
391 | } | |
17df50a5 XL |
392 | } |
393 | } | |
394 | } | |
395 | } | |
396 | if intel_syntax { | |
397 | template_str.push_str("\n.att_syntax\n"); | |
398 | } | |
399 | ||
a1dfa0c6 | 400 | unsafe { |
353b0b11 | 401 | llvm::LLVMAppendModuleInlineAsm( |
17df50a5 XL |
402 | self.llmod, |
403 | template_str.as_ptr().cast(), | |
404 | template_str.len(), | |
405 | ); | |
a1dfa0c6 | 406 | } |
cc61c64b XL |
407 | } |
408 | } | |
532ac7d7 | 409 | |
a2a8927a XL |
410 | pub(crate) fn inline_asm_call<'ll>( |
411 | bx: &mut Builder<'_, 'll, '_>, | |
ba9703b0 XL |
412 | asm: &str, |
413 | cons: &str, | |
532ac7d7 XL |
414 | inputs: &[&'ll Value], |
415 | output: &'ll llvm::Type, | |
416 | volatile: bool, | |
417 | alignstack: bool, | |
5099ac24 | 418 | dia: llvm::AsmDialect, |
f9f354fc | 419 | line_spans: &[Span], |
a2a8927a XL |
420 | unwind: bool, |
421 | dest_catch_funclet: Option<( | |
422 | &'ll llvm::BasicBlock, | |
423 | &'ll llvm::BasicBlock, | |
424 | Option<&Funclet<'ll>>, | |
425 | )>, | |
532ac7d7 | 426 | ) -> Option<&'ll Value> { |
dfeec247 XL |
427 | let volatile = if volatile { llvm::True } else { llvm::False }; |
428 | let alignstack = if alignstack { llvm::True } else { llvm::False }; | |
a2a8927a | 429 | let can_throw = if unwind { llvm::True } else { llvm::False }; |
dfeec247 XL |
430 | |
431 | let argtys = inputs | |
432 | .iter() | |
433 | .map(|v| { | |
434 | debug!("Asm Input Type: {:?}", *v); | |
435 | bx.cx.val_ty(*v) | |
436 | }) | |
437 | .collect::<Vec<_>>(); | |
532ac7d7 XL |
438 | |
439 | debug!("Asm Output Type: {:?}", output); | |
a2a8927a | 440 | let fty = bx.cx.type_func(&argtys, output); |
532ac7d7 XL |
441 | unsafe { |
442 | // Ask LLVM to verify that the constraints are well-formed. | |
ba9703b0 | 443 | let constraints_ok = llvm::LLVMRustInlineAsmVerify(fty, cons.as_ptr().cast(), cons.len()); |
416331ca | 444 | debug!("constraint verification result: {:?}", constraints_ok); |
532ac7d7 XL |
445 | if constraints_ok { |
446 | let v = llvm::LLVMRustInlineAsm( | |
447 | fty, | |
ba9703b0 XL |
448 | asm.as_ptr().cast(), |
449 | asm.len(), | |
450 | cons.as_ptr().cast(), | |
451 | cons.len(), | |
532ac7d7 XL |
452 | volatile, |
453 | alignstack, | |
5099ac24 | 454 | dia, |
a2a8927a | 455 | can_throw, |
532ac7d7 | 456 | ); |
a2a8927a XL |
457 | |
458 | let call = if let Some((dest, catch, funclet)) = dest_catch_funclet { | |
49aad941 | 459 | bx.invoke(fty, None, None, v, inputs, dest, catch, funclet) |
a2a8927a | 460 | } else { |
49aad941 | 461 | bx.call(fty, None, None, v, inputs, None) |
a2a8927a | 462 | }; |
f9f354fc XL |
463 | |
464 | // Store mark in a metadata node so we can map LLVM errors | |
9c376795 | 465 | // back to source locations. See #17552. |
f9f354fc XL |
466 | let key = "srcloc"; |
467 | let kind = llvm::LLVMGetMDKindIDInContext( | |
468 | bx.llcx, | |
469 | key.as_ptr() as *const c_char, | |
470 | key.len() as c_uint, | |
471 | ); | |
472 | ||
473 | // srcloc contains one integer for each line of assembly code. | |
474 | // Unfortunately this isn't enough to encode a full span so instead | |
475 | // we just encode the start position of each line. | |
476 | // FIXME: Figure out a way to pass the entire line spans. | |
477 | let mut srcloc = vec![]; | |
5099ac24 | 478 | if dia == llvm::AsmDialect::Intel && line_spans.len() > 1 { |
f9f354fc XL |
479 | // LLVM inserts an extra line to add the ".intel_syntax", so add |
480 | // a dummy srcloc entry for it. | |
481 | // | |
482 | // Don't do this if we only have 1 line span since that may be | |
483 | // due to the asm template string coming from a macro. LLVM will | |
484 | // default to the first srcloc for lines that don't have an | |
485 | // associated srcloc. | |
486 | srcloc.push(bx.const_i32(0)); | |
487 | } | |
488 | srcloc.extend(line_spans.iter().map(|span| bx.const_i32(span.lo().to_u32() as i32))); | |
489 | let md = llvm::LLVMMDNodeInContext(bx.llcx, srcloc.as_ptr(), srcloc.len() as u32); | |
490 | llvm::LLVMSetMetadata(call, kind, md); | |
491 | ||
492 | Some(call) | |
532ac7d7 XL |
493 | } else { |
494 | // LLVM has detected an issue with our constraints, bail out | |
495 | None | |
496 | } | |
497 | } | |
498 | } | |
f9f354fc | 499 | |
3dfed10e XL |
500 | /// If the register is an xmm/ymm/zmm register then return its index. |
501 | fn xmm_reg_index(reg: InlineAsmReg) -> Option<u32> { | |
502 | match reg { | |
503 | InlineAsmReg::X86(reg) | |
504 | if reg as u32 >= X86InlineAsmReg::xmm0 as u32 | |
505 | && reg as u32 <= X86InlineAsmReg::xmm15 as u32 => | |
506 | { | |
507 | Some(reg as u32 - X86InlineAsmReg::xmm0 as u32) | |
508 | } | |
509 | InlineAsmReg::X86(reg) | |
510 | if reg as u32 >= X86InlineAsmReg::ymm0 as u32 | |
511 | && reg as u32 <= X86InlineAsmReg::ymm15 as u32 => | |
512 | { | |
513 | Some(reg as u32 - X86InlineAsmReg::ymm0 as u32) | |
514 | } | |
515 | InlineAsmReg::X86(reg) | |
516 | if reg as u32 >= X86InlineAsmReg::zmm0 as u32 | |
517 | && reg as u32 <= X86InlineAsmReg::zmm31 as u32 => | |
518 | { | |
519 | Some(reg as u32 - X86InlineAsmReg::zmm0 as u32) | |
520 | } | |
521 | _ => None, | |
522 | } | |
523 | } | |
524 | ||
487cf647 FG |
525 | /// If the register is an AArch64 integer register then return its index. |
526 | fn a64_reg_index(reg: InlineAsmReg) -> Option<u32> { | |
527 | match reg { | |
528 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x0) => Some(0), | |
529 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x1) => Some(1), | |
530 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x2) => Some(2), | |
531 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x3) => Some(3), | |
532 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x4) => Some(4), | |
533 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x5) => Some(5), | |
534 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x6) => Some(6), | |
535 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x7) => Some(7), | |
536 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x8) => Some(8), | |
537 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x9) => Some(9), | |
538 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x10) => Some(10), | |
539 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x11) => Some(11), | |
540 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x12) => Some(12), | |
541 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x13) => Some(13), | |
542 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x14) => Some(14), | |
543 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x15) => Some(15), | |
544 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x16) => Some(16), | |
545 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x17) => Some(17), | |
546 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x18) => Some(18), | |
547 | // x19 is reserved | |
548 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x20) => Some(20), | |
549 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x21) => Some(21), | |
550 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x22) => Some(22), | |
551 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x23) => Some(23), | |
552 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x24) => Some(24), | |
553 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x25) => Some(25), | |
554 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x26) => Some(26), | |
555 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x27) => Some(27), | |
556 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x28) => Some(28), | |
557 | // x29 is reserved | |
558 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x30) => Some(30), | |
559 | _ => None, | |
560 | } | |
561 | } | |
562 | ||
3dfed10e XL |
563 | /// If the register is an AArch64 vector register then return its index. |
564 | fn a64_vreg_index(reg: InlineAsmReg) -> Option<u32> { | |
565 | match reg { | |
566 | InlineAsmReg::AArch64(reg) | |
567 | if reg as u32 >= AArch64InlineAsmReg::v0 as u32 | |
568 | && reg as u32 <= AArch64InlineAsmReg::v31 as u32 => | |
569 | { | |
570 | Some(reg as u32 - AArch64InlineAsmReg::v0 as u32) | |
571 | } | |
572 | _ => None, | |
573 | } | |
574 | } | |
575 | ||
f9f354fc | 576 | /// Converts a register class to an LLVM constraint code. |
a2a8927a | 577 | fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> String { |
f9f354fc | 578 | match reg { |
3dfed10e XL |
579 | // For vector registers LLVM wants the register name to match the type size. |
580 | InlineAsmRegOrRegClass::Reg(reg) => { | |
581 | if let Some(idx) = xmm_reg_index(reg) { | |
582 | let class = if let Some(layout) = layout { | |
583 | match layout.size.bytes() { | |
584 | 64 => 'z', | |
585 | 32 => 'y', | |
586 | _ => 'x', | |
587 | } | |
588 | } else { | |
589 | // We use f32 as the type for discarded outputs | |
590 | 'x' | |
591 | }; | |
592 | format!("{{{}mm{}}}", class, idx) | |
487cf647 FG |
593 | } else if let Some(idx) = a64_reg_index(reg) { |
594 | let class = if let Some(layout) = layout { | |
595 | match layout.size.bytes() { | |
596 | 8 => 'x', | |
597 | _ => 'w', | |
598 | } | |
599 | } else { | |
600 | // We use i32 as the type for discarded outputs | |
601 | 'w' | |
602 | }; | |
603 | if class == 'x' && reg == InlineAsmReg::AArch64(AArch64InlineAsmReg::x30) { | |
604 | // LLVM doesn't recognize x30. use lr instead. | |
605 | "{lr}".to_string() | |
606 | } else { | |
607 | format!("{{{}{}}}", class, idx) | |
608 | } | |
3dfed10e XL |
609 | } else if let Some(idx) = a64_vreg_index(reg) { |
610 | let class = if let Some(layout) = layout { | |
611 | match layout.size.bytes() { | |
612 | 16 => 'q', | |
613 | 8 => 'd', | |
614 | 4 => 's', | |
615 | 2 => 'h', | |
616 | 1 => 'd', // We fixup i8 to i8x8 | |
617 | _ => unreachable!(), | |
618 | } | |
619 | } else { | |
620 | // We use i64x2 as the type for discarded outputs | |
621 | 'q' | |
622 | }; | |
623 | format!("{{{}{}}}", class, idx) | |
6a06907d XL |
624 | } else if reg == InlineAsmReg::Arm(ArmInlineAsmReg::r14) { |
625 | // LLVM doesn't recognize r14 | |
626 | "{lr}".to_string() | |
3dfed10e XL |
627 | } else { |
628 | format!("{{{}}}", reg.name()) | |
629 | } | |
630 | } | |
2b03887a FG |
631 | // The constraints can be retrieved from |
632 | // https://llvm.org/docs/LangRef.html#supported-constraint-code-list | |
f9f354fc XL |
633 | InlineAsmRegOrRegClass::RegClass(reg) => match reg { |
634 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => "r", | |
635 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) => "w", | |
636 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => "x", | |
136023e0 XL |
637 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => { |
638 | unreachable!("clobber-only") | |
639 | } | |
f9f354fc | 640 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) => "r", |
f9f354fc XL |
641 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg) |
642 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16) | |
643 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8) => "t", | |
644 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16) | |
645 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8) | |
646 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4) => "x", | |
647 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg) | |
648 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg) => "w", | |
f035d41b | 649 | InlineAsmRegClass::Hexagon(HexagonInlineAsmRegClass::reg) => "r", |
49aad941 FG |
650 | InlineAsmRegClass::LoongArch(LoongArchInlineAsmRegClass::reg) => "r", |
651 | InlineAsmRegClass::LoongArch(LoongArchInlineAsmRegClass::freg) => "f", | |
1b1a35ee XL |
652 | InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg) => "r", |
653 | InlineAsmRegClass::Mips(MipsInlineAsmRegClass::freg) => "f", | |
f9f354fc XL |
654 | InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg16) => "h", |
655 | InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg32) => "r", | |
656 | InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg64) => "l", | |
17df50a5 XL |
657 | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg) => "r", |
658 | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => "b", | |
659 | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::freg) => "f", | |
94222f64 XL |
660 | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::cr) |
661 | | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::xer) => { | |
662 | unreachable!("clobber-only") | |
663 | } | |
f9f354fc XL |
664 | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) => "r", |
665 | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => "f", | |
136023e0 XL |
666 | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::vreg) => { |
667 | unreachable!("clobber-only") | |
668 | } | |
f9f354fc XL |
669 | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg) => "r", |
670 | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => "Q", | |
671 | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_byte) => "q", | |
672 | InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg) | |
673 | | InlineAsmRegClass::X86(X86InlineAsmRegClass::ymm_reg) => "x", | |
674 | InlineAsmRegClass::X86(X86InlineAsmRegClass::zmm_reg) => "v", | |
675 | InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => "^Yk", | |
136023e0 | 676 | InlineAsmRegClass::X86( |
04454e1e FG |
677 | X86InlineAsmRegClass::x87_reg |
678 | | X86InlineAsmRegClass::mmx_reg | |
923072b8 FG |
679 | | X86InlineAsmRegClass::kreg0 |
680 | | X86InlineAsmRegClass::tmm_reg, | |
136023e0 | 681 | ) => unreachable!("clobber-only"), |
fc512014 | 682 | InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => "r", |
17df50a5 XL |
683 | InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::reg) => "r", |
684 | InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::wreg) => "w", | |
a2a8927a XL |
685 | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg) => "r", |
686 | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_upper) => "d", | |
687 | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_pair) => "r", | |
688 | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_iw) => "w", | |
689 | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_ptr) => "e", | |
94222f64 XL |
690 | InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg) => "r", |
691 | InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => "f", | |
5099ac24 | 692 | InlineAsmRegClass::Msp430(Msp430InlineAsmRegClass::reg) => "r", |
353b0b11 FG |
693 | InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg) => "r", |
694 | InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg_addr) => "a", | |
695 | InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg_data) => "d", | |
29967ef6 XL |
696 | InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { |
697 | bug!("LLVM backend does not support SPIR-V") | |
698 | } | |
6a06907d | 699 | InlineAsmRegClass::Err => unreachable!(), |
f9f354fc XL |
700 | } |
701 | .to_string(), | |
702 | } | |
703 | } | |
704 | ||
705 | /// Converts a modifier into LLVM's equivalent modifier. | |
706 | fn modifier_to_llvm( | |
707 | arch: InlineAsmArch, | |
708 | reg: InlineAsmRegClass, | |
709 | modifier: Option<char>, | |
710 | ) -> Option<char> { | |
2b03887a FG |
711 | // The modifiers can be retrieved from |
712 | // https://llvm.org/docs/LangRef.html#asm-template-argument-modifiers | |
f9f354fc XL |
713 | match reg { |
714 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => modifier, | |
715 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) | |
716 | | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => { | |
717 | if modifier == Some('v') { None } else { modifier } | |
718 | } | |
136023e0 XL |
719 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => { |
720 | unreachable!("clobber-only") | |
721 | } | |
a2a8927a | 722 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) => None, |
f9f354fc XL |
723 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg) |
724 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16) => None, | |
725 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg) | |
726 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16) | |
727 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8) => Some('P'), | |
728 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg) | |
729 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8) | |
730 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4) => { | |
731 | if modifier.is_none() { | |
732 | Some('q') | |
733 | } else { | |
734 | modifier | |
735 | } | |
736 | } | |
f035d41b | 737 | InlineAsmRegClass::Hexagon(_) => None, |
49aad941 | 738 | InlineAsmRegClass::LoongArch(_) => None, |
1b1a35ee | 739 | InlineAsmRegClass::Mips(_) => None, |
f9f354fc | 740 | InlineAsmRegClass::Nvptx(_) => None, |
17df50a5 | 741 | InlineAsmRegClass::PowerPC(_) => None, |
f9f354fc XL |
742 | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) |
743 | | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => None, | |
136023e0 XL |
744 | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::vreg) => { |
745 | unreachable!("clobber-only") | |
746 | } | |
f9f354fc XL |
747 | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg) |
748 | | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => match modifier { | |
749 | None if arch == InlineAsmArch::X86_64 => Some('q'), | |
750 | None => Some('k'), | |
751 | Some('l') => Some('b'), | |
752 | Some('h') => Some('h'), | |
753 | Some('x') => Some('w'), | |
754 | Some('e') => Some('k'), | |
755 | Some('r') => Some('q'), | |
756 | _ => unreachable!(), | |
757 | }, | |
758 | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_byte) => None, | |
759 | InlineAsmRegClass::X86(reg @ X86InlineAsmRegClass::xmm_reg) | |
760 | | InlineAsmRegClass::X86(reg @ X86InlineAsmRegClass::ymm_reg) | |
761 | | InlineAsmRegClass::X86(reg @ X86InlineAsmRegClass::zmm_reg) => match (reg, modifier) { | |
762 | (X86InlineAsmRegClass::xmm_reg, None) => Some('x'), | |
763 | (X86InlineAsmRegClass::ymm_reg, None) => Some('t'), | |
764 | (X86InlineAsmRegClass::zmm_reg, None) => Some('g'), | |
765 | (_, Some('x')) => Some('x'), | |
766 | (_, Some('y')) => Some('t'), | |
767 | (_, Some('z')) => Some('g'), | |
768 | _ => unreachable!(), | |
769 | }, | |
770 | InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => None, | |
04454e1e FG |
771 | InlineAsmRegClass::X86( |
772 | X86InlineAsmRegClass::x87_reg | |
773 | | X86InlineAsmRegClass::mmx_reg | |
923072b8 FG |
774 | | X86InlineAsmRegClass::kreg0 |
775 | | X86InlineAsmRegClass::tmm_reg, | |
04454e1e | 776 | ) => { |
136023e0 XL |
777 | unreachable!("clobber-only") |
778 | } | |
fc512014 | 779 | InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => None, |
17df50a5 | 780 | InlineAsmRegClass::Bpf(_) => None, |
a2a8927a XL |
781 | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_pair) |
782 | | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_iw) | |
783 | | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_ptr) => match modifier { | |
784 | Some('h') => Some('B'), | |
785 | Some('l') => Some('A'), | |
786 | _ => None, | |
787 | }, | |
788 | InlineAsmRegClass::Avr(_) => None, | |
94222f64 | 789 | InlineAsmRegClass::S390x(_) => None, |
5099ac24 | 790 | InlineAsmRegClass::Msp430(_) => None, |
29967ef6 XL |
791 | InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { |
792 | bug!("LLVM backend does not support SPIR-V") | |
793 | } | |
353b0b11 | 794 | InlineAsmRegClass::M68k(_) => None, |
6a06907d | 795 | InlineAsmRegClass::Err => unreachable!(), |
f9f354fc XL |
796 | } |
797 | } | |
798 | ||
799 | /// Type to use for outputs that are discarded. It doesn't really matter what | |
800 | /// the type is, as long as it is valid for the constraint code. | |
a2a8927a | 801 | fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &'ll Type { |
f9f354fc XL |
802 | match reg { |
803 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => cx.type_i32(), | |
804 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) | |
805 | | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => { | |
806 | cx.type_vector(cx.type_i64(), 2) | |
807 | } | |
136023e0 XL |
808 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => { |
809 | unreachable!("clobber-only") | |
810 | } | |
a2a8927a | 811 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) => cx.type_i32(), |
f9f354fc XL |
812 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg) |
813 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16) => cx.type_f32(), | |
814 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg) | |
815 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16) | |
816 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8) => cx.type_f64(), | |
817 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg) | |
818 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8) | |
819 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4) => { | |
820 | cx.type_vector(cx.type_i64(), 2) | |
821 | } | |
f035d41b | 822 | InlineAsmRegClass::Hexagon(HexagonInlineAsmRegClass::reg) => cx.type_i32(), |
49aad941 FG |
823 | InlineAsmRegClass::LoongArch(LoongArchInlineAsmRegClass::reg) => cx.type_i32(), |
824 | InlineAsmRegClass::LoongArch(LoongArchInlineAsmRegClass::freg) => cx.type_f32(), | |
1b1a35ee XL |
825 | InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg) => cx.type_i32(), |
826 | InlineAsmRegClass::Mips(MipsInlineAsmRegClass::freg) => cx.type_f32(), | |
f9f354fc XL |
827 | InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg16) => cx.type_i16(), |
828 | InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg32) => cx.type_i32(), | |
829 | InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg64) => cx.type_i64(), | |
17df50a5 XL |
830 | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg) => cx.type_i32(), |
831 | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => cx.type_i32(), | |
832 | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::freg) => cx.type_f64(), | |
94222f64 XL |
833 | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::cr) |
834 | | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::xer) => { | |
835 | unreachable!("clobber-only") | |
836 | } | |
f9f354fc XL |
837 | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) => cx.type_i32(), |
838 | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => cx.type_f32(), | |
136023e0 XL |
839 | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::vreg) => { |
840 | unreachable!("clobber-only") | |
841 | } | |
f9f354fc XL |
842 | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg) |
843 | | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => cx.type_i32(), | |
844 | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_byte) => cx.type_i8(), | |
845 | InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg) | |
846 | | InlineAsmRegClass::X86(X86InlineAsmRegClass::ymm_reg) | |
847 | | InlineAsmRegClass::X86(X86InlineAsmRegClass::zmm_reg) => cx.type_f32(), | |
848 | InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => cx.type_i16(), | |
04454e1e FG |
849 | InlineAsmRegClass::X86( |
850 | X86InlineAsmRegClass::x87_reg | |
851 | | X86InlineAsmRegClass::mmx_reg | |
923072b8 FG |
852 | | X86InlineAsmRegClass::kreg0 |
853 | | X86InlineAsmRegClass::tmm_reg, | |
04454e1e | 854 | ) => { |
136023e0 XL |
855 | unreachable!("clobber-only") |
856 | } | |
fc512014 | 857 | InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => cx.type_i32(), |
17df50a5 XL |
858 | InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::reg) => cx.type_i64(), |
859 | InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::wreg) => cx.type_i32(), | |
a2a8927a XL |
860 | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg) => cx.type_i8(), |
861 | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_upper) => cx.type_i8(), | |
862 | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_pair) => cx.type_i16(), | |
863 | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_iw) => cx.type_i16(), | |
864 | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_ptr) => cx.type_i16(), | |
94222f64 XL |
865 | InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg) => cx.type_i32(), |
866 | InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => cx.type_f64(), | |
5099ac24 | 867 | InlineAsmRegClass::Msp430(Msp430InlineAsmRegClass::reg) => cx.type_i16(), |
353b0b11 FG |
868 | InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg) => cx.type_i32(), |
869 | InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg_addr) => cx.type_i32(), | |
870 | InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg_data) => cx.type_i32(), | |
29967ef6 XL |
871 | InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { |
872 | bug!("LLVM backend does not support SPIR-V") | |
873 | } | |
6a06907d | 874 | InlineAsmRegClass::Err => unreachable!(), |
f9f354fc XL |
875 | } |
876 | } | |
877 | ||
878 | /// Helper function to get the LLVM type for a Scalar. Pointers are returned as | |
879 | /// the equivalent integer type. | |
a2a8927a | 880 | fn llvm_asm_scalar_type<'ll>(cx: &CodegenCx<'ll, '_>, scalar: Scalar) -> &'ll Type { |
9ffffee4 | 881 | let dl = &cx.tcx.data_layout; |
04454e1e | 882 | match scalar.primitive() { |
f9f354fc XL |
883 | Primitive::Int(Integer::I8, _) => cx.type_i8(), |
884 | Primitive::Int(Integer::I16, _) => cx.type_i16(), | |
885 | Primitive::Int(Integer::I32, _) => cx.type_i32(), | |
886 | Primitive::Int(Integer::I64, _) => cx.type_i64(), | |
887 | Primitive::F32 => cx.type_f32(), | |
888 | Primitive::F64 => cx.type_f64(), | |
9ffffee4 FG |
889 | // FIXME(erikdesjardins): handle non-default addrspace ptr sizes |
890 | Primitive::Pointer(_) => cx.type_from_integer(dl.ptr_sized_integer()), | |
f9f354fc XL |
891 | _ => unreachable!(), |
892 | } | |
893 | } | |
894 | ||
895 | /// Fix up an input value to work around LLVM bugs. | |
a2a8927a XL |
896 | fn llvm_fixup_input<'ll, 'tcx>( |
897 | bx: &mut Builder<'_, 'll, 'tcx>, | |
f9f354fc XL |
898 | mut value: &'ll Value, |
899 | reg: InlineAsmRegClass, | |
900 | layout: &TyAndLayout<'tcx>, | |
901 | ) -> &'ll Value { | |
9ffffee4 | 902 | let dl = &bx.tcx.data_layout; |
c295e0f8 | 903 | match (reg, layout.abi) { |
f9f354fc | 904 | (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => { |
04454e1e | 905 | if let Primitive::Int(Integer::I8, _) = s.primitive() { |
f9f354fc XL |
906 | let vec_ty = bx.cx.type_vector(bx.cx.type_i8(), 8); |
907 | bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0)) | |
908 | } else { | |
909 | value | |
910 | } | |
911 | } | |
912 | (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) => { | |
913 | let elem_ty = llvm_asm_scalar_type(bx.cx, s); | |
914 | let count = 16 / layout.size.bytes(); | |
915 | let vec_ty = bx.cx.type_vector(elem_ty, count); | |
9ffffee4 FG |
916 | // FIXME(erikdesjardins): handle non-default addrspace ptr sizes |
917 | if let Primitive::Pointer(_) = s.primitive() { | |
918 | let t = bx.type_from_integer(dl.ptr_sized_integer()); | |
919 | value = bx.ptrtoint(value, t); | |
f9f354fc XL |
920 | } |
921 | bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0)) | |
922 | } | |
923 | ( | |
924 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), | |
925 | Abi::Vector { element, count }, | |
926 | ) if layout.size.bytes() == 8 => { | |
927 | let elem_ty = llvm_asm_scalar_type(bx.cx, element); | |
c295e0f8 | 928 | let vec_ty = bx.cx.type_vector(elem_ty, count); |
f9f354fc XL |
929 | let indices: Vec<_> = (0..count * 2).map(|x| bx.const_i32(x as i32)).collect(); |
930 | bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices)) | |
931 | } | |
932 | (InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd), Abi::Scalar(s)) | |
04454e1e | 933 | if s.primitive() == Primitive::F64 => |
f9f354fc XL |
934 | { |
935 | bx.bitcast(value, bx.cx.type_i64()) | |
936 | } | |
937 | ( | |
938 | InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg), | |
939 | Abi::Vector { .. }, | |
940 | ) if layout.size.bytes() == 64 => bx.bitcast(value, bx.cx.type_vector(bx.cx.type_f64(), 8)), | |
3dfed10e XL |
941 | ( |
942 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16), | |
943 | Abi::Scalar(s), | |
944 | ) => { | |
04454e1e | 945 | if let Primitive::Int(Integer::I32, _) = s.primitive() { |
3dfed10e XL |
946 | bx.bitcast(value, bx.cx.type_f32()) |
947 | } else { | |
948 | value | |
949 | } | |
950 | } | |
f9f354fc XL |
951 | ( |
952 | InlineAsmRegClass::Arm( | |
3dfed10e | 953 | ArmInlineAsmRegClass::dreg |
f9f354fc | 954 | | ArmInlineAsmRegClass::dreg_low8 |
3dfed10e | 955 | | ArmInlineAsmRegClass::dreg_low16, |
f9f354fc XL |
956 | ), |
957 | Abi::Scalar(s), | |
958 | ) => { | |
04454e1e | 959 | if let Primitive::Int(Integer::I64, _) = s.primitive() { |
3dfed10e | 960 | bx.bitcast(value, bx.cx.type_f64()) |
f9f354fc XL |
961 | } else { |
962 | value | |
963 | } | |
964 | } | |
04454e1e FG |
965 | (InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg), Abi::Scalar(s)) => { |
966 | match s.primitive() { | |
967 | // MIPS only supports register-length arithmetics. | |
968 | Primitive::Int(Integer::I8 | Integer::I16, _) => bx.zext(value, bx.cx.type_i32()), | |
969 | Primitive::F32 => bx.bitcast(value, bx.cx.type_i32()), | |
970 | Primitive::F64 => bx.bitcast(value, bx.cx.type_i64()), | |
971 | _ => value, | |
972 | } | |
973 | } | |
f9f354fc XL |
974 | _ => value, |
975 | } | |
976 | } | |
977 | ||
978 | /// Fix up an output value to work around LLVM bugs. | |
a2a8927a XL |
979 | fn llvm_fixup_output<'ll, 'tcx>( |
980 | bx: &mut Builder<'_, 'll, 'tcx>, | |
f9f354fc XL |
981 | mut value: &'ll Value, |
982 | reg: InlineAsmRegClass, | |
983 | layout: &TyAndLayout<'tcx>, | |
984 | ) -> &'ll Value { | |
c295e0f8 | 985 | match (reg, layout.abi) { |
f9f354fc | 986 | (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => { |
04454e1e | 987 | if let Primitive::Int(Integer::I8, _) = s.primitive() { |
f9f354fc XL |
988 | bx.extract_element(value, bx.const_i32(0)) |
989 | } else { | |
990 | value | |
991 | } | |
992 | } | |
993 | (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) => { | |
994 | value = bx.extract_element(value, bx.const_i32(0)); | |
9ffffee4 | 995 | if let Primitive::Pointer(_) = s.primitive() { |
f9f354fc XL |
996 | value = bx.inttoptr(value, layout.llvm_type(bx.cx)); |
997 | } | |
998 | value | |
999 | } | |
1000 | ( | |
1001 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), | |
1002 | Abi::Vector { element, count }, | |
1003 | ) if layout.size.bytes() == 8 => { | |
1004 | let elem_ty = llvm_asm_scalar_type(bx.cx, element); | |
c295e0f8 XL |
1005 | let vec_ty = bx.cx.type_vector(elem_ty, count * 2); |
1006 | let indices: Vec<_> = (0..count).map(|x| bx.const_i32(x as i32)).collect(); | |
f9f354fc XL |
1007 | bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices)) |
1008 | } | |
1009 | (InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd), Abi::Scalar(s)) | |
04454e1e | 1010 | if s.primitive() == Primitive::F64 => |
f9f354fc XL |
1011 | { |
1012 | bx.bitcast(value, bx.cx.type_f64()) | |
1013 | } | |
1014 | ( | |
1015 | InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg), | |
1016 | Abi::Vector { .. }, | |
1017 | ) if layout.size.bytes() == 64 => bx.bitcast(value, layout.llvm_type(bx.cx)), | |
3dfed10e XL |
1018 | ( |
1019 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16), | |
1020 | Abi::Scalar(s), | |
1021 | ) => { | |
04454e1e | 1022 | if let Primitive::Int(Integer::I32, _) = s.primitive() { |
3dfed10e XL |
1023 | bx.bitcast(value, bx.cx.type_i32()) |
1024 | } else { | |
1025 | value | |
1026 | } | |
1027 | } | |
f9f354fc XL |
1028 | ( |
1029 | InlineAsmRegClass::Arm( | |
3dfed10e | 1030 | ArmInlineAsmRegClass::dreg |
f9f354fc | 1031 | | ArmInlineAsmRegClass::dreg_low8 |
3dfed10e | 1032 | | ArmInlineAsmRegClass::dreg_low16, |
f9f354fc XL |
1033 | ), |
1034 | Abi::Scalar(s), | |
1035 | ) => { | |
04454e1e | 1036 | if let Primitive::Int(Integer::I64, _) = s.primitive() { |
3dfed10e | 1037 | bx.bitcast(value, bx.cx.type_i64()) |
f9f354fc XL |
1038 | } else { |
1039 | value | |
1040 | } | |
1041 | } | |
04454e1e FG |
1042 | (InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg), Abi::Scalar(s)) => { |
1043 | match s.primitive() { | |
1044 | // MIPS only supports register-length arithmetics. | |
1045 | Primitive::Int(Integer::I8, _) => bx.trunc(value, bx.cx.type_i8()), | |
1046 | Primitive::Int(Integer::I16, _) => bx.trunc(value, bx.cx.type_i16()), | |
1047 | Primitive::F32 => bx.bitcast(value, bx.cx.type_f32()), | |
1048 | Primitive::F64 => bx.bitcast(value, bx.cx.type_f64()), | |
1049 | _ => value, | |
1050 | } | |
1051 | } | |
f9f354fc XL |
1052 | _ => value, |
1053 | } | |
1054 | } | |
1055 | ||
1056 | /// Output type to use for llvm_fixup_output. | |
a2a8927a | 1057 | fn llvm_fixup_output_type<'ll, 'tcx>( |
f9f354fc XL |
1058 | cx: &CodegenCx<'ll, 'tcx>, |
1059 | reg: InlineAsmRegClass, | |
1060 | layout: &TyAndLayout<'tcx>, | |
1061 | ) -> &'ll Type { | |
c295e0f8 | 1062 | match (reg, layout.abi) { |
f9f354fc | 1063 | (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => { |
04454e1e | 1064 | if let Primitive::Int(Integer::I8, _) = s.primitive() { |
f9f354fc XL |
1065 | cx.type_vector(cx.type_i8(), 8) |
1066 | } else { | |
1067 | layout.llvm_type(cx) | |
1068 | } | |
1069 | } | |
1070 | (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) => { | |
1071 | let elem_ty = llvm_asm_scalar_type(cx, s); | |
1072 | let count = 16 / layout.size.bytes(); | |
1073 | cx.type_vector(elem_ty, count) | |
1074 | } | |
1075 | ( | |
1076 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), | |
1077 | Abi::Vector { element, count }, | |
1078 | ) if layout.size.bytes() == 8 => { | |
1079 | let elem_ty = llvm_asm_scalar_type(cx, element); | |
1080 | cx.type_vector(elem_ty, count * 2) | |
1081 | } | |
1082 | (InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd), Abi::Scalar(s)) | |
04454e1e | 1083 | if s.primitive() == Primitive::F64 => |
f9f354fc XL |
1084 | { |
1085 | cx.type_i64() | |
1086 | } | |
1087 | ( | |
1088 | InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg), | |
1089 | Abi::Vector { .. }, | |
1090 | ) if layout.size.bytes() == 64 => cx.type_vector(cx.type_f64(), 8), | |
3dfed10e XL |
1091 | ( |
1092 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16), | |
1093 | Abi::Scalar(s), | |
1094 | ) => { | |
04454e1e | 1095 | if let Primitive::Int(Integer::I32, _) = s.primitive() { |
3dfed10e XL |
1096 | cx.type_f32() |
1097 | } else { | |
1098 | layout.llvm_type(cx) | |
1099 | } | |
1100 | } | |
f9f354fc XL |
1101 | ( |
1102 | InlineAsmRegClass::Arm( | |
3dfed10e | 1103 | ArmInlineAsmRegClass::dreg |
f9f354fc | 1104 | | ArmInlineAsmRegClass::dreg_low8 |
3dfed10e | 1105 | | ArmInlineAsmRegClass::dreg_low16, |
f9f354fc XL |
1106 | ), |
1107 | Abi::Scalar(s), | |
1108 | ) => { | |
04454e1e | 1109 | if let Primitive::Int(Integer::I64, _) = s.primitive() { |
3dfed10e | 1110 | cx.type_f64() |
f9f354fc XL |
1111 | } else { |
1112 | layout.llvm_type(cx) | |
1113 | } | |
1114 | } | |
04454e1e FG |
1115 | (InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg), Abi::Scalar(s)) => { |
1116 | match s.primitive() { | |
1117 | // MIPS only supports register-length arithmetics. | |
1118 | Primitive::Int(Integer::I8 | Integer::I16, _) => cx.type_i32(), | |
1119 | Primitive::F32 => cx.type_i32(), | |
1120 | Primitive::F64 => cx.type_i64(), | |
1121 | _ => layout.llvm_type(cx), | |
1122 | } | |
1123 | } | |
f9f354fc XL |
1124 | _ => layout.llvm_type(cx), |
1125 | } | |
1126 | } |