]>
Commit | Line | Data |
---|---|---|
5e7ed085 | 1 | use crate::attributes; |
dfeec247 | 2 | use crate::builder::Builder; |
a2a8927a | 3 | use crate::common::Funclet; |
9fa01778 | 4 | use crate::context::CodegenCx; |
dfeec247 | 5 | use crate::llvm; |
f9f354fc | 6 | use crate::type_::Type; |
9fa01778 | 7 | use crate::type_of::LayoutLlvmExt; |
9fa01778 | 8 | use crate::value::Value; |
54a0048b | 9 | |
3dfed10e | 10 | use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece}; |
a1dfa0c6 | 11 | use rustc_codegen_ssa::mir::operand::OperandValue; |
dfeec247 | 12 | use rustc_codegen_ssa::traits::*; |
f9f354fc | 13 | use rustc_data_structures::fx::FxHashMap; |
f9f354fc | 14 | use rustc_middle::ty::layout::TyAndLayout; |
3c0e092e | 15 | use rustc_middle::{bug, span_bug, ty::Instance}; |
5099ac24 | 16 | use rustc_span::{Pos, Span}; |
f9f354fc XL |
17 | use rustc_target::abi::*; |
18 | use rustc_target::asm::*; | |
32a655c1 | 19 | |
dfeec247 | 20 | use libc::{c_char, c_uint}; |
5e7ed085 | 21 | use smallvec::SmallVec; |
a1dfa0c6 | 22 | |
a2a8927a | 23 | impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { |
f9f354fc XL |
24 | fn codegen_inline_asm( |
25 | &mut self, | |
26 | template: &[InlineAsmTemplatePiece], | |
27 | operands: &[InlineAsmOperandRef<'tcx, Self>], | |
28 | options: InlineAsmOptions, | |
29 | line_spans: &[Span], | |
3c0e092e | 30 | instance: Instance<'_>, |
a2a8927a | 31 | dest_catch_funclet: Option<(Self::BasicBlock, Self::BasicBlock, Option<&Self::Funclet>)>, |
f9f354fc XL |
32 | ) { |
33 | let asm_arch = self.tcx.sess.asm_arch.unwrap(); | |
54a0048b | 34 | |
f9f354fc XL |
35 | // Collect the types of output operands |
36 | let mut constraints = vec![]; | |
cdc7bbd5 | 37 | let mut clobbers = vec![]; |
f9f354fc XL |
38 | let mut output_types = vec![]; |
39 | let mut op_idx = FxHashMap::default(); | |
136023e0 | 40 | let mut clobbered_x87 = false; |
f9f354fc XL |
41 | for (idx, op) in operands.iter().enumerate() { |
42 | match *op { | |
43 | InlineAsmOperandRef::Out { reg, late, place } => { | |
cdc7bbd5 XL |
44 | let is_target_supported = |reg_class: InlineAsmRegClass| { |
45 | for &(_, feature) in reg_class.supported_types(asm_arch) { | |
46 | if let Some(feature) = feature { | |
3c0e092e | 47 | let codegen_fn_attrs = self.tcx.codegen_fn_attrs(instance.def_id()); |
5099ac24 FG |
48 | if self.tcx.sess.target_features.contains(&feature) |
49 | || codegen_fn_attrs.target_features.contains(&feature) | |
cdc7bbd5 XL |
50 | { |
51 | return true; | |
52 | } | |
53 | } else { | |
54 | // Register class is unconditionally supported | |
55 | return true; | |
56 | } | |
57 | } | |
58 | false | |
59 | }; | |
60 | ||
3dfed10e XL |
61 | let mut layout = None; |
62 | let ty = if let Some(ref place) = place { | |
63 | layout = Some(&place.layout); | |
f9f354fc | 64 | llvm_fixup_output_type(self.cx, reg.reg_class(), &place.layout) |
136023e0 XL |
65 | } else if matches!( |
66 | reg.reg_class(), | |
67 | InlineAsmRegClass::X86( | |
68 | X86InlineAsmRegClass::mmx_reg | X86InlineAsmRegClass::x87_reg | |
69 | ) | |
70 | ) { | |
71 | // Special handling for x87/mmx registers: we always | |
72 | // clobber the whole set if one register is marked as | |
73 | // clobbered. This is due to the way LLVM handles the | |
74 | // FP stack in inline assembly. | |
75 | if !clobbered_x87 { | |
76 | clobbered_x87 = true; | |
77 | clobbers.push("~{st}".to_string()); | |
78 | for i in 1..=7 { | |
79 | clobbers.push(format!("~{{st({})}}", i)); | |
80 | } | |
81 | } | |
82 | continue; | |
83 | } else if !is_target_supported(reg.reg_class()) | |
84 | || reg.reg_class().is_clobber_only(asm_arch) | |
85 | { | |
cdc7bbd5 XL |
86 | // We turn discarded outputs into clobber constraints |
87 | // if the target feature needed by the register class is | |
88 | // disabled. This is necessary otherwise LLVM will try | |
89 | // to actually allocate a register for the dummy output. | |
90 | assert!(matches!(reg, InlineAsmRegOrRegClass::Reg(_))); | |
91 | clobbers.push(format!("~{}", reg_to_llvm(reg, None))); | |
92 | continue; | |
f9f354fc XL |
93 | } else { |
94 | // If the output is discarded, we don't really care what | |
95 | // type is used. We're just using this to tell LLVM to | |
96 | // reserve the register. | |
97 | dummy_output_type(self.cx, reg.reg_class()) | |
98 | }; | |
99 | output_types.push(ty); | |
100 | op_idx.insert(idx, constraints.len()); | |
101 | let prefix = if late { "=" } else { "=&" }; | |
3dfed10e | 102 | constraints.push(format!("{}{}", prefix, reg_to_llvm(reg, layout))); |
f9f354fc XL |
103 | } |
104 | InlineAsmOperandRef::InOut { reg, late, in_value, out_place } => { | |
3dfed10e XL |
105 | let layout = if let Some(ref out_place) = out_place { |
106 | &out_place.layout | |
f9f354fc XL |
107 | } else { |
108 | // LLVM required tied operands to have the same type, | |
109 | // so we just use the type of the input. | |
3dfed10e | 110 | &in_value.layout |
f9f354fc | 111 | }; |
3dfed10e | 112 | let ty = llvm_fixup_output_type(self.cx, reg.reg_class(), layout); |
f9f354fc XL |
113 | output_types.push(ty); |
114 | op_idx.insert(idx, constraints.len()); | |
115 | let prefix = if late { "=" } else { "=&" }; | |
3dfed10e | 116 | constraints.push(format!("{}{}", prefix, reg_to_llvm(reg, Some(layout)))); |
f9f354fc XL |
117 | } |
118 | _ => {} | |
119 | } | |
a1dfa0c6 | 120 | } |
0bf4aa26 | 121 | |
f9f354fc XL |
122 | // Collect input operands |
123 | let mut inputs = vec![]; | |
124 | for (idx, op) in operands.iter().enumerate() { | |
125 | match *op { | |
126 | InlineAsmOperandRef::In { reg, value } => { | |
3dfed10e | 127 | let llval = |
f9f354fc | 128 | llvm_fixup_input(self, value.immediate(), reg.reg_class(), &value.layout); |
3dfed10e | 129 | inputs.push(llval); |
f9f354fc | 130 | op_idx.insert(idx, constraints.len()); |
3dfed10e | 131 | constraints.push(reg_to_llvm(reg, Some(&value.layout))); |
f9f354fc | 132 | } |
487cf647 | 133 | InlineAsmOperandRef::InOut { reg, late, in_value, out_place: _ } => { |
f9f354fc XL |
134 | let value = llvm_fixup_input( |
135 | self, | |
136 | in_value.immediate(), | |
137 | reg.reg_class(), | |
138 | &in_value.layout, | |
139 | ); | |
140 | inputs.push(value); | |
487cf647 FG |
141 | |
142 | // In the case of fixed registers, we have the choice of | |
143 | // either using a tied operand or duplicating the constraint. | |
144 | // We prefer the latter because it matches the behavior of | |
145 | // Clang. | |
146 | if late && matches!(reg, InlineAsmRegOrRegClass::Reg(_)) { | |
9c376795 | 147 | constraints.push(reg_to_llvm(reg, Some(&in_value.layout)).to_string()); |
487cf647 FG |
148 | } else { |
149 | constraints.push(format!("{}", op_idx[&idx])); | |
150 | } | |
f9f354fc XL |
151 | } |
152 | InlineAsmOperandRef::SymFn { instance } => { | |
153 | inputs.push(self.cx.get_fn(instance)); | |
154 | op_idx.insert(idx, constraints.len()); | |
155 | constraints.push("s".to_string()); | |
156 | } | |
157 | InlineAsmOperandRef::SymStatic { def_id } => { | |
158 | inputs.push(self.cx.get_static(def_id)); | |
159 | op_idx.insert(idx, constraints.len()); | |
160 | constraints.push("s".to_string()); | |
161 | } | |
162 | _ => {} | |
163 | } | |
164 | } | |
165 | ||
166 | // Build the template string | |
167 | let mut template_str = String::new(); | |
168 | for piece in template { | |
169 | match *piece { | |
170 | InlineAsmTemplatePiece::String(ref s) => { | |
171 | if s.contains('$') { | |
172 | for c in s.chars() { | |
173 | if c == '$' { | |
174 | template_str.push_str("$$"); | |
175 | } else { | |
176 | template_str.push(c); | |
177 | } | |
178 | } | |
179 | } else { | |
180 | template_str.push_str(s) | |
181 | } | |
182 | } | |
183 | InlineAsmTemplatePiece::Placeholder { operand_idx, modifier, span: _ } => { | |
184 | match operands[operand_idx] { | |
185 | InlineAsmOperandRef::In { reg, .. } | |
186 | | InlineAsmOperandRef::Out { reg, .. } | |
187 | | InlineAsmOperandRef::InOut { reg, .. } => { | |
188 | let modifier = modifier_to_llvm(asm_arch, reg.reg_class(), modifier); | |
189 | if let Some(modifier) = modifier { | |
190 | template_str.push_str(&format!( | |
191 | "${{{}:{}}}", | |
192 | op_idx[&operand_idx], modifier | |
193 | )); | |
194 | } else { | |
195 | template_str.push_str(&format!("${{{}}}", op_idx[&operand_idx])); | |
196 | } | |
197 | } | |
198 | InlineAsmOperandRef::Const { ref string } => { | |
199 | // Const operands get injected directly into the template | |
200 | template_str.push_str(string); | |
201 | } | |
202 | InlineAsmOperandRef::SymFn { .. } | |
203 | | InlineAsmOperandRef::SymStatic { .. } => { | |
204 | // Only emit the raw symbol name | |
205 | template_str.push_str(&format!("${{{}:c}}", op_idx[&operand_idx])); | |
206 | } | |
207 | } | |
208 | } | |
209 | } | |
210 | } | |
211 | ||
cdc7bbd5 | 212 | constraints.append(&mut clobbers); |
f9f354fc XL |
213 | if !options.contains(InlineAsmOptions::PRESERVES_FLAGS) { |
214 | match asm_arch { | |
215 | InlineAsmArch::AArch64 | InlineAsmArch::Arm => { | |
216 | constraints.push("~{cc}".to_string()); | |
217 | } | |
218 | InlineAsmArch::X86 | InlineAsmArch::X86_64 => { | |
219 | constraints.extend_from_slice(&[ | |
220 | "~{dirflag}".to_string(), | |
221 | "~{fpsr}".to_string(), | |
222 | "~{flags}".to_string(), | |
223 | ]); | |
224 | } | |
94222f64 XL |
225 | InlineAsmArch::RiscV32 | InlineAsmArch::RiscV64 => { |
226 | constraints.extend_from_slice(&[ | |
227 | "~{vtype}".to_string(), | |
228 | "~{vl}".to_string(), | |
229 | "~{vxsat}".to_string(), | |
230 | "~{vxrm}".to_string(), | |
231 | ]); | |
232 | } | |
a2a8927a XL |
233 | InlineAsmArch::Avr => { |
234 | constraints.push("~{sreg}".to_string()); | |
235 | } | |
f9f354fc | 236 | InlineAsmArch::Nvptx64 => {} |
17df50a5 | 237 | InlineAsmArch::PowerPC | InlineAsmArch::PowerPC64 => {} |
f035d41b | 238 | InlineAsmArch::Hexagon => {} |
29967ef6 | 239 | InlineAsmArch::Mips | InlineAsmArch::Mips64 => {} |
94222f64 | 240 | InlineAsmArch::S390x => {} |
29967ef6 | 241 | InlineAsmArch::SpirV => {} |
3c0e092e | 242 | InlineAsmArch::Wasm32 | InlineAsmArch::Wasm64 => {} |
17df50a5 | 243 | InlineAsmArch::Bpf => {} |
5099ac24 FG |
244 | InlineAsmArch::Msp430 => { |
245 | constraints.push("~{sr}".to_string()); | |
246 | } | |
f9f354fc XL |
247 | } |
248 | } | |
249 | if !options.contains(InlineAsmOptions::NOMEM) { | |
250 | // This is actually ignored by LLVM, but it's probably best to keep | |
251 | // it just in case. LLVM instead uses the ReadOnly/ReadNone | |
252 | // attributes on the call instruction to optimize. | |
253 | constraints.push("~{memory}".to_string()); | |
254 | } | |
255 | let volatile = !options.contains(InlineAsmOptions::PURE); | |
256 | let alignstack = !options.contains(InlineAsmOptions::NOSTACK); | |
257 | let output_type = match &output_types[..] { | |
258 | [] => self.type_void(), | |
259 | [ty] => ty, | |
c295e0f8 | 260 | tys => self.type_struct(tys, false), |
f9f354fc XL |
261 | }; |
262 | let dialect = match asm_arch { | |
263 | InlineAsmArch::X86 | InlineAsmArch::X86_64 | |
264 | if !options.contains(InlineAsmOptions::ATT_SYNTAX) => | |
265 | { | |
5099ac24 | 266 | llvm::AsmDialect::Intel |
f9f354fc | 267 | } |
5099ac24 | 268 | _ => llvm::AsmDialect::Att, |
f9f354fc XL |
269 | }; |
270 | let result = inline_asm_call( | |
271 | self, | |
272 | &template_str, | |
273 | &constraints.join(","), | |
274 | &inputs, | |
275 | output_type, | |
276 | volatile, | |
277 | alignstack, | |
278 | dialect, | |
279 | line_spans, | |
a2a8927a XL |
280 | options.contains(InlineAsmOptions::MAY_UNWIND), |
281 | dest_catch_funclet, | |
f9f354fc XL |
282 | ) |
283 | .unwrap_or_else(|| span_bug!(line_spans[0], "LLVM asm constraint validation failed")); | |
284 | ||
5e7ed085 | 285 | let mut attrs = SmallVec::<[_; 2]>::new(); |
f9f354fc XL |
286 | if options.contains(InlineAsmOptions::PURE) { |
287 | if options.contains(InlineAsmOptions::NOMEM) { | |
487cf647 | 288 | attrs.push(llvm::MemoryEffects::None.create_attr(self.cx.llcx)); |
f9f354fc | 289 | } else if options.contains(InlineAsmOptions::READONLY) { |
487cf647 | 290 | attrs.push(llvm::MemoryEffects::ReadOnly.create_attr(self.cx.llcx)); |
f9f354fc | 291 | } |
5e7ed085 | 292 | attrs.push(llvm::AttributeKind::WillReturn.create_attr(self.cx.llcx)); |
29967ef6 | 293 | } else if options.contains(InlineAsmOptions::NOMEM) { |
487cf647 | 294 | attrs.push(llvm::MemoryEffects::InaccessibleMemOnly.create_attr(self.cx.llcx)); |
f9f354fc | 295 | } else { |
29967ef6 | 296 | // LLVM doesn't have an attribute to represent ReadOnly + SideEffect |
f9f354fc | 297 | } |
5e7ed085 | 298 | attributes::apply_to_callsite(result, llvm::AttributePlace::Function, &{ attrs }); |
f9f354fc | 299 | |
04454e1e FG |
300 | // Switch to the 'normal' basic block if we did an `invoke` instead of a `call` |
301 | if let Some((dest, _, _)) = dest_catch_funclet { | |
302 | self.switch_to_block(dest); | |
303 | } | |
304 | ||
f9f354fc XL |
305 | // Write results to outputs |
306 | for (idx, op) in operands.iter().enumerate() { | |
307 | if let InlineAsmOperandRef::Out { reg, place: Some(place), .. } | |
308 | | InlineAsmOperandRef::InOut { reg, out_place: Some(place), .. } = *op | |
309 | { | |
310 | let value = if output_types.len() == 1 { | |
311 | result | |
312 | } else { | |
313 | self.extract_value(result, op_idx[&idx] as u64) | |
314 | }; | |
315 | let value = llvm_fixup_output(self, value, reg.reg_class(), &place.layout); | |
316 | OperandValue::Immediate(value).store(self, place); | |
317 | } | |
318 | } | |
a1dfa0c6 | 319 | } |
54a0048b | 320 | } |
cc61c64b | 321 | |
04454e1e | 322 | impl<'tcx> AsmMethods<'tcx> for CodegenCx<'_, 'tcx> { |
17df50a5 XL |
323 | fn codegen_global_asm( |
324 | &self, | |
325 | template: &[InlineAsmTemplatePiece], | |
04454e1e | 326 | operands: &[GlobalAsmOperandRef<'tcx>], |
17df50a5 XL |
327 | options: InlineAsmOptions, |
328 | _line_spans: &[Span], | |
329 | ) { | |
330 | let asm_arch = self.tcx.sess.asm_arch.unwrap(); | |
331 | ||
332 | // Default to Intel syntax on x86 | |
333 | let intel_syntax = matches!(asm_arch, InlineAsmArch::X86 | InlineAsmArch::X86_64) | |
334 | && !options.contains(InlineAsmOptions::ATT_SYNTAX); | |
335 | ||
336 | // Build the template string | |
337 | let mut template_str = String::new(); | |
338 | if intel_syntax { | |
339 | template_str.push_str(".intel_syntax\n"); | |
340 | } | |
341 | for piece in template { | |
342 | match *piece { | |
343 | InlineAsmTemplatePiece::String(ref s) => template_str.push_str(s), | |
344 | InlineAsmTemplatePiece::Placeholder { operand_idx, modifier: _, span: _ } => { | |
345 | match operands[operand_idx] { | |
346 | GlobalAsmOperandRef::Const { ref string } => { | |
347 | // Const operands get injected directly into the | |
348 | // template. Note that we don't need to escape $ | |
349 | // here unlike normal inline assembly. | |
350 | template_str.push_str(string); | |
351 | } | |
04454e1e FG |
352 | GlobalAsmOperandRef::SymFn { instance } => { |
353 | let llval = self.get_fn(instance); | |
354 | self.add_compiler_used_global(llval); | |
355 | let symbol = llvm::build_string(|s| unsafe { | |
356 | llvm::LLVMRustGetMangledName(llval, s); | |
357 | }) | |
358 | .expect("symbol is not valid UTF-8"); | |
359 | template_str.push_str(&symbol); | |
360 | } | |
361 | GlobalAsmOperandRef::SymStatic { def_id } => { | |
362 | let llval = self | |
363 | .renamed_statics | |
364 | .borrow() | |
365 | .get(&def_id) | |
366 | .copied() | |
367 | .unwrap_or_else(|| self.get_static(def_id)); | |
368 | self.add_compiler_used_global(llval); | |
369 | let symbol = llvm::build_string(|s| unsafe { | |
370 | llvm::LLVMRustGetMangledName(llval, s); | |
371 | }) | |
372 | .expect("symbol is not valid UTF-8"); | |
373 | template_str.push_str(&symbol); | |
374 | } | |
17df50a5 XL |
375 | } |
376 | } | |
377 | } | |
378 | } | |
379 | if intel_syntax { | |
380 | template_str.push_str("\n.att_syntax\n"); | |
381 | } | |
382 | ||
a1dfa0c6 | 383 | unsafe { |
17df50a5 XL |
384 | llvm::LLVMRustAppendModuleInlineAsm( |
385 | self.llmod, | |
386 | template_str.as_ptr().cast(), | |
387 | template_str.len(), | |
388 | ); | |
a1dfa0c6 | 389 | } |
cc61c64b XL |
390 | } |
391 | } | |
532ac7d7 | 392 | |
a2a8927a XL |
393 | pub(crate) fn inline_asm_call<'ll>( |
394 | bx: &mut Builder<'_, 'll, '_>, | |
ba9703b0 XL |
395 | asm: &str, |
396 | cons: &str, | |
532ac7d7 XL |
397 | inputs: &[&'ll Value], |
398 | output: &'ll llvm::Type, | |
399 | volatile: bool, | |
400 | alignstack: bool, | |
5099ac24 | 401 | dia: llvm::AsmDialect, |
f9f354fc | 402 | line_spans: &[Span], |
a2a8927a XL |
403 | unwind: bool, |
404 | dest_catch_funclet: Option<( | |
405 | &'ll llvm::BasicBlock, | |
406 | &'ll llvm::BasicBlock, | |
407 | Option<&Funclet<'ll>>, | |
408 | )>, | |
532ac7d7 | 409 | ) -> Option<&'ll Value> { |
dfeec247 XL |
410 | let volatile = if volatile { llvm::True } else { llvm::False }; |
411 | let alignstack = if alignstack { llvm::True } else { llvm::False }; | |
a2a8927a | 412 | let can_throw = if unwind { llvm::True } else { llvm::False }; |
dfeec247 XL |
413 | |
414 | let argtys = inputs | |
415 | .iter() | |
416 | .map(|v| { | |
417 | debug!("Asm Input Type: {:?}", *v); | |
418 | bx.cx.val_ty(*v) | |
419 | }) | |
420 | .collect::<Vec<_>>(); | |
532ac7d7 XL |
421 | |
422 | debug!("Asm Output Type: {:?}", output); | |
a2a8927a | 423 | let fty = bx.cx.type_func(&argtys, output); |
532ac7d7 XL |
424 | unsafe { |
425 | // Ask LLVM to verify that the constraints are well-formed. | |
ba9703b0 | 426 | let constraints_ok = llvm::LLVMRustInlineAsmVerify(fty, cons.as_ptr().cast(), cons.len()); |
416331ca | 427 | debug!("constraint verification result: {:?}", constraints_ok); |
532ac7d7 XL |
428 | if constraints_ok { |
429 | let v = llvm::LLVMRustInlineAsm( | |
430 | fty, | |
ba9703b0 XL |
431 | asm.as_ptr().cast(), |
432 | asm.len(), | |
433 | cons.as_ptr().cast(), | |
434 | cons.len(), | |
532ac7d7 XL |
435 | volatile, |
436 | alignstack, | |
5099ac24 | 437 | dia, |
a2a8927a | 438 | can_throw, |
532ac7d7 | 439 | ); |
a2a8927a XL |
440 | |
441 | let call = if let Some((dest, catch, funclet)) = dest_catch_funclet { | |
2b03887a | 442 | bx.invoke(fty, None, v, inputs, dest, catch, funclet) |
a2a8927a | 443 | } else { |
2b03887a | 444 | bx.call(fty, None, v, inputs, None) |
a2a8927a | 445 | }; |
f9f354fc XL |
446 | |
447 | // Store mark in a metadata node so we can map LLVM errors | |
9c376795 | 448 | // back to source locations. See #17552. |
f9f354fc XL |
449 | let key = "srcloc"; |
450 | let kind = llvm::LLVMGetMDKindIDInContext( | |
451 | bx.llcx, | |
452 | key.as_ptr() as *const c_char, | |
453 | key.len() as c_uint, | |
454 | ); | |
455 | ||
456 | // srcloc contains one integer for each line of assembly code. | |
457 | // Unfortunately this isn't enough to encode a full span so instead | |
458 | // we just encode the start position of each line. | |
459 | // FIXME: Figure out a way to pass the entire line spans. | |
460 | let mut srcloc = vec![]; | |
5099ac24 | 461 | if dia == llvm::AsmDialect::Intel && line_spans.len() > 1 { |
f9f354fc XL |
462 | // LLVM inserts an extra line to add the ".intel_syntax", so add |
463 | // a dummy srcloc entry for it. | |
464 | // | |
465 | // Don't do this if we only have 1 line span since that may be | |
466 | // due to the asm template string coming from a macro. LLVM will | |
467 | // default to the first srcloc for lines that don't have an | |
468 | // associated srcloc. | |
469 | srcloc.push(bx.const_i32(0)); | |
470 | } | |
471 | srcloc.extend(line_spans.iter().map(|span| bx.const_i32(span.lo().to_u32() as i32))); | |
472 | let md = llvm::LLVMMDNodeInContext(bx.llcx, srcloc.as_ptr(), srcloc.len() as u32); | |
473 | llvm::LLVMSetMetadata(call, kind, md); | |
474 | ||
475 | Some(call) | |
532ac7d7 XL |
476 | } else { |
477 | // LLVM has detected an issue with our constraints, bail out | |
478 | None | |
479 | } | |
480 | } | |
481 | } | |
f9f354fc | 482 | |
3dfed10e XL |
483 | /// If the register is an xmm/ymm/zmm register then return its index. |
484 | fn xmm_reg_index(reg: InlineAsmReg) -> Option<u32> { | |
485 | match reg { | |
486 | InlineAsmReg::X86(reg) | |
487 | if reg as u32 >= X86InlineAsmReg::xmm0 as u32 | |
488 | && reg as u32 <= X86InlineAsmReg::xmm15 as u32 => | |
489 | { | |
490 | Some(reg as u32 - X86InlineAsmReg::xmm0 as u32) | |
491 | } | |
492 | InlineAsmReg::X86(reg) | |
493 | if reg as u32 >= X86InlineAsmReg::ymm0 as u32 | |
494 | && reg as u32 <= X86InlineAsmReg::ymm15 as u32 => | |
495 | { | |
496 | Some(reg as u32 - X86InlineAsmReg::ymm0 as u32) | |
497 | } | |
498 | InlineAsmReg::X86(reg) | |
499 | if reg as u32 >= X86InlineAsmReg::zmm0 as u32 | |
500 | && reg as u32 <= X86InlineAsmReg::zmm31 as u32 => | |
501 | { | |
502 | Some(reg as u32 - X86InlineAsmReg::zmm0 as u32) | |
503 | } | |
504 | _ => None, | |
505 | } | |
506 | } | |
507 | ||
487cf647 FG |
508 | /// If the register is an AArch64 integer register then return its index. |
509 | fn a64_reg_index(reg: InlineAsmReg) -> Option<u32> { | |
510 | match reg { | |
511 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x0) => Some(0), | |
512 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x1) => Some(1), | |
513 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x2) => Some(2), | |
514 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x3) => Some(3), | |
515 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x4) => Some(4), | |
516 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x5) => Some(5), | |
517 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x6) => Some(6), | |
518 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x7) => Some(7), | |
519 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x8) => Some(8), | |
520 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x9) => Some(9), | |
521 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x10) => Some(10), | |
522 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x11) => Some(11), | |
523 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x12) => Some(12), | |
524 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x13) => Some(13), | |
525 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x14) => Some(14), | |
526 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x15) => Some(15), | |
527 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x16) => Some(16), | |
528 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x17) => Some(17), | |
529 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x18) => Some(18), | |
530 | // x19 is reserved | |
531 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x20) => Some(20), | |
532 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x21) => Some(21), | |
533 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x22) => Some(22), | |
534 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x23) => Some(23), | |
535 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x24) => Some(24), | |
536 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x25) => Some(25), | |
537 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x26) => Some(26), | |
538 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x27) => Some(27), | |
539 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x28) => Some(28), | |
540 | // x29 is reserved | |
541 | InlineAsmReg::AArch64(AArch64InlineAsmReg::x30) => Some(30), | |
542 | _ => None, | |
543 | } | |
544 | } | |
545 | ||
3dfed10e XL |
546 | /// If the register is an AArch64 vector register then return its index. |
547 | fn a64_vreg_index(reg: InlineAsmReg) -> Option<u32> { | |
548 | match reg { | |
549 | InlineAsmReg::AArch64(reg) | |
550 | if reg as u32 >= AArch64InlineAsmReg::v0 as u32 | |
551 | && reg as u32 <= AArch64InlineAsmReg::v31 as u32 => | |
552 | { | |
553 | Some(reg as u32 - AArch64InlineAsmReg::v0 as u32) | |
554 | } | |
555 | _ => None, | |
556 | } | |
557 | } | |
558 | ||
f9f354fc | 559 | /// Converts a register class to an LLVM constraint code. |
a2a8927a | 560 | fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> String { |
f9f354fc | 561 | match reg { |
3dfed10e XL |
562 | // For vector registers LLVM wants the register name to match the type size. |
563 | InlineAsmRegOrRegClass::Reg(reg) => { | |
564 | if let Some(idx) = xmm_reg_index(reg) { | |
565 | let class = if let Some(layout) = layout { | |
566 | match layout.size.bytes() { | |
567 | 64 => 'z', | |
568 | 32 => 'y', | |
569 | _ => 'x', | |
570 | } | |
571 | } else { | |
572 | // We use f32 as the type for discarded outputs | |
573 | 'x' | |
574 | }; | |
575 | format!("{{{}mm{}}}", class, idx) | |
487cf647 FG |
576 | } else if let Some(idx) = a64_reg_index(reg) { |
577 | let class = if let Some(layout) = layout { | |
578 | match layout.size.bytes() { | |
579 | 8 => 'x', | |
580 | _ => 'w', | |
581 | } | |
582 | } else { | |
583 | // We use i32 as the type for discarded outputs | |
584 | 'w' | |
585 | }; | |
586 | if class == 'x' && reg == InlineAsmReg::AArch64(AArch64InlineAsmReg::x30) { | |
587 | // LLVM doesn't recognize x30. use lr instead. | |
588 | "{lr}".to_string() | |
589 | } else { | |
590 | format!("{{{}{}}}", class, idx) | |
591 | } | |
3dfed10e XL |
592 | } else if let Some(idx) = a64_vreg_index(reg) { |
593 | let class = if let Some(layout) = layout { | |
594 | match layout.size.bytes() { | |
595 | 16 => 'q', | |
596 | 8 => 'd', | |
597 | 4 => 's', | |
598 | 2 => 'h', | |
599 | 1 => 'd', // We fixup i8 to i8x8 | |
600 | _ => unreachable!(), | |
601 | } | |
602 | } else { | |
603 | // We use i64x2 as the type for discarded outputs | |
604 | 'q' | |
605 | }; | |
606 | format!("{{{}{}}}", class, idx) | |
6a06907d XL |
607 | } else if reg == InlineAsmReg::Arm(ArmInlineAsmReg::r14) { |
608 | // LLVM doesn't recognize r14 | |
609 | "{lr}".to_string() | |
3dfed10e XL |
610 | } else { |
611 | format!("{{{}}}", reg.name()) | |
612 | } | |
613 | } | |
2b03887a FG |
614 | // The constraints can be retrieved from |
615 | // https://llvm.org/docs/LangRef.html#supported-constraint-code-list | |
f9f354fc XL |
616 | InlineAsmRegOrRegClass::RegClass(reg) => match reg { |
617 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => "r", | |
618 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) => "w", | |
619 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => "x", | |
136023e0 XL |
620 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => { |
621 | unreachable!("clobber-only") | |
622 | } | |
f9f354fc | 623 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) => "r", |
f9f354fc XL |
624 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg) |
625 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16) | |
626 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8) => "t", | |
627 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16) | |
628 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8) | |
629 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4) => "x", | |
630 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg) | |
631 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg) => "w", | |
f035d41b | 632 | InlineAsmRegClass::Hexagon(HexagonInlineAsmRegClass::reg) => "r", |
1b1a35ee XL |
633 | InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg) => "r", |
634 | InlineAsmRegClass::Mips(MipsInlineAsmRegClass::freg) => "f", | |
f9f354fc XL |
635 | InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg16) => "h", |
636 | InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg32) => "r", | |
637 | InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg64) => "l", | |
17df50a5 XL |
638 | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg) => "r", |
639 | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => "b", | |
640 | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::freg) => "f", | |
94222f64 XL |
641 | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::cr) |
642 | | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::xer) => { | |
643 | unreachable!("clobber-only") | |
644 | } | |
f9f354fc XL |
645 | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) => "r", |
646 | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => "f", | |
136023e0 XL |
647 | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::vreg) => { |
648 | unreachable!("clobber-only") | |
649 | } | |
f9f354fc XL |
650 | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg) => "r", |
651 | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => "Q", | |
652 | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_byte) => "q", | |
653 | InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg) | |
654 | | InlineAsmRegClass::X86(X86InlineAsmRegClass::ymm_reg) => "x", | |
655 | InlineAsmRegClass::X86(X86InlineAsmRegClass::zmm_reg) => "v", | |
656 | InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => "^Yk", | |
136023e0 | 657 | InlineAsmRegClass::X86( |
04454e1e FG |
658 | X86InlineAsmRegClass::x87_reg |
659 | | X86InlineAsmRegClass::mmx_reg | |
923072b8 FG |
660 | | X86InlineAsmRegClass::kreg0 |
661 | | X86InlineAsmRegClass::tmm_reg, | |
136023e0 | 662 | ) => unreachable!("clobber-only"), |
fc512014 | 663 | InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => "r", |
17df50a5 XL |
664 | InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::reg) => "r", |
665 | InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::wreg) => "w", | |
a2a8927a XL |
666 | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg) => "r", |
667 | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_upper) => "d", | |
668 | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_pair) => "r", | |
669 | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_iw) => "w", | |
670 | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_ptr) => "e", | |
94222f64 XL |
671 | InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg) => "r", |
672 | InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => "f", | |
5099ac24 | 673 | InlineAsmRegClass::Msp430(Msp430InlineAsmRegClass::reg) => "r", |
29967ef6 XL |
674 | InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { |
675 | bug!("LLVM backend does not support SPIR-V") | |
676 | } | |
6a06907d | 677 | InlineAsmRegClass::Err => unreachable!(), |
f9f354fc XL |
678 | } |
679 | .to_string(), | |
680 | } | |
681 | } | |
682 | ||
683 | /// Converts a modifier into LLVM's equivalent modifier. | |
684 | fn modifier_to_llvm( | |
685 | arch: InlineAsmArch, | |
686 | reg: InlineAsmRegClass, | |
687 | modifier: Option<char>, | |
688 | ) -> Option<char> { | |
2b03887a FG |
689 | // The modifiers can be retrieved from |
690 | // https://llvm.org/docs/LangRef.html#asm-template-argument-modifiers | |
f9f354fc XL |
691 | match reg { |
692 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => modifier, | |
693 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) | |
694 | | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => { | |
695 | if modifier == Some('v') { None } else { modifier } | |
696 | } | |
136023e0 XL |
697 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => { |
698 | unreachable!("clobber-only") | |
699 | } | |
a2a8927a | 700 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) => None, |
f9f354fc XL |
701 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg) |
702 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16) => None, | |
703 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg) | |
704 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16) | |
705 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8) => Some('P'), | |
706 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg) | |
707 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8) | |
708 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4) => { | |
709 | if modifier.is_none() { | |
710 | Some('q') | |
711 | } else { | |
712 | modifier | |
713 | } | |
714 | } | |
f035d41b | 715 | InlineAsmRegClass::Hexagon(_) => None, |
1b1a35ee | 716 | InlineAsmRegClass::Mips(_) => None, |
f9f354fc | 717 | InlineAsmRegClass::Nvptx(_) => None, |
17df50a5 | 718 | InlineAsmRegClass::PowerPC(_) => None, |
f9f354fc XL |
719 | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) |
720 | | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => None, | |
136023e0 XL |
721 | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::vreg) => { |
722 | unreachable!("clobber-only") | |
723 | } | |
f9f354fc XL |
724 | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg) |
725 | | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => match modifier { | |
726 | None if arch == InlineAsmArch::X86_64 => Some('q'), | |
727 | None => Some('k'), | |
728 | Some('l') => Some('b'), | |
729 | Some('h') => Some('h'), | |
730 | Some('x') => Some('w'), | |
731 | Some('e') => Some('k'), | |
732 | Some('r') => Some('q'), | |
733 | _ => unreachable!(), | |
734 | }, | |
735 | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_byte) => None, | |
736 | InlineAsmRegClass::X86(reg @ X86InlineAsmRegClass::xmm_reg) | |
737 | | InlineAsmRegClass::X86(reg @ X86InlineAsmRegClass::ymm_reg) | |
738 | | InlineAsmRegClass::X86(reg @ X86InlineAsmRegClass::zmm_reg) => match (reg, modifier) { | |
739 | (X86InlineAsmRegClass::xmm_reg, None) => Some('x'), | |
740 | (X86InlineAsmRegClass::ymm_reg, None) => Some('t'), | |
741 | (X86InlineAsmRegClass::zmm_reg, None) => Some('g'), | |
742 | (_, Some('x')) => Some('x'), | |
743 | (_, Some('y')) => Some('t'), | |
744 | (_, Some('z')) => Some('g'), | |
745 | _ => unreachable!(), | |
746 | }, | |
747 | InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => None, | |
04454e1e FG |
748 | InlineAsmRegClass::X86( |
749 | X86InlineAsmRegClass::x87_reg | |
750 | | X86InlineAsmRegClass::mmx_reg | |
923072b8 FG |
751 | | X86InlineAsmRegClass::kreg0 |
752 | | X86InlineAsmRegClass::tmm_reg, | |
04454e1e | 753 | ) => { |
136023e0 XL |
754 | unreachable!("clobber-only") |
755 | } | |
fc512014 | 756 | InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => None, |
17df50a5 | 757 | InlineAsmRegClass::Bpf(_) => None, |
a2a8927a XL |
758 | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_pair) |
759 | | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_iw) | |
760 | | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_ptr) => match modifier { | |
761 | Some('h') => Some('B'), | |
762 | Some('l') => Some('A'), | |
763 | _ => None, | |
764 | }, | |
765 | InlineAsmRegClass::Avr(_) => None, | |
94222f64 | 766 | InlineAsmRegClass::S390x(_) => None, |
5099ac24 | 767 | InlineAsmRegClass::Msp430(_) => None, |
29967ef6 XL |
768 | InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { |
769 | bug!("LLVM backend does not support SPIR-V") | |
770 | } | |
6a06907d | 771 | InlineAsmRegClass::Err => unreachable!(), |
f9f354fc XL |
772 | } |
773 | } | |
774 | ||
775 | /// Type to use for outputs that are discarded. It doesn't really matter what | |
776 | /// the type is, as long as it is valid for the constraint code. | |
a2a8927a | 777 | fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &'ll Type { |
f9f354fc XL |
778 | match reg { |
779 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => cx.type_i32(), | |
780 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) | |
781 | | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => { | |
782 | cx.type_vector(cx.type_i64(), 2) | |
783 | } | |
136023e0 XL |
784 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => { |
785 | unreachable!("clobber-only") | |
786 | } | |
a2a8927a | 787 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) => cx.type_i32(), |
f9f354fc XL |
788 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg) |
789 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16) => cx.type_f32(), | |
790 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg) | |
791 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16) | |
792 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8) => cx.type_f64(), | |
793 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg) | |
794 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8) | |
795 | | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4) => { | |
796 | cx.type_vector(cx.type_i64(), 2) | |
797 | } | |
f035d41b | 798 | InlineAsmRegClass::Hexagon(HexagonInlineAsmRegClass::reg) => cx.type_i32(), |
1b1a35ee XL |
799 | InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg) => cx.type_i32(), |
800 | InlineAsmRegClass::Mips(MipsInlineAsmRegClass::freg) => cx.type_f32(), | |
f9f354fc XL |
801 | InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg16) => cx.type_i16(), |
802 | InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg32) => cx.type_i32(), | |
803 | InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg64) => cx.type_i64(), | |
17df50a5 XL |
804 | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg) => cx.type_i32(), |
805 | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => cx.type_i32(), | |
806 | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::freg) => cx.type_f64(), | |
94222f64 XL |
807 | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::cr) |
808 | | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::xer) => { | |
809 | unreachable!("clobber-only") | |
810 | } | |
f9f354fc XL |
811 | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) => cx.type_i32(), |
812 | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => cx.type_f32(), | |
136023e0 XL |
813 | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::vreg) => { |
814 | unreachable!("clobber-only") | |
815 | } | |
f9f354fc XL |
816 | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg) |
817 | | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => cx.type_i32(), | |
818 | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_byte) => cx.type_i8(), | |
819 | InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg) | |
820 | | InlineAsmRegClass::X86(X86InlineAsmRegClass::ymm_reg) | |
821 | | InlineAsmRegClass::X86(X86InlineAsmRegClass::zmm_reg) => cx.type_f32(), | |
822 | InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => cx.type_i16(), | |
04454e1e FG |
823 | InlineAsmRegClass::X86( |
824 | X86InlineAsmRegClass::x87_reg | |
825 | | X86InlineAsmRegClass::mmx_reg | |
923072b8 FG |
826 | | X86InlineAsmRegClass::kreg0 |
827 | | X86InlineAsmRegClass::tmm_reg, | |
04454e1e | 828 | ) => { |
136023e0 XL |
829 | unreachable!("clobber-only") |
830 | } | |
fc512014 | 831 | InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => cx.type_i32(), |
17df50a5 XL |
832 | InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::reg) => cx.type_i64(), |
833 | InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::wreg) => cx.type_i32(), | |
a2a8927a XL |
834 | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg) => cx.type_i8(), |
835 | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_upper) => cx.type_i8(), | |
836 | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_pair) => cx.type_i16(), | |
837 | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_iw) => cx.type_i16(), | |
838 | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_ptr) => cx.type_i16(), | |
94222f64 XL |
839 | InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg) => cx.type_i32(), |
840 | InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => cx.type_f64(), | |
5099ac24 | 841 | InlineAsmRegClass::Msp430(Msp430InlineAsmRegClass::reg) => cx.type_i16(), |
29967ef6 XL |
842 | InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { |
843 | bug!("LLVM backend does not support SPIR-V") | |
844 | } | |
6a06907d | 845 | InlineAsmRegClass::Err => unreachable!(), |
f9f354fc XL |
846 | } |
847 | } | |
848 | ||
849 | /// Helper function to get the LLVM type for a Scalar. Pointers are returned as | |
850 | /// the equivalent integer type. | |
a2a8927a | 851 | fn llvm_asm_scalar_type<'ll>(cx: &CodegenCx<'ll, '_>, scalar: Scalar) -> &'ll Type { |
04454e1e | 852 | match scalar.primitive() { |
f9f354fc XL |
853 | Primitive::Int(Integer::I8, _) => cx.type_i8(), |
854 | Primitive::Int(Integer::I16, _) => cx.type_i16(), | |
855 | Primitive::Int(Integer::I32, _) => cx.type_i32(), | |
856 | Primitive::Int(Integer::I64, _) => cx.type_i64(), | |
857 | Primitive::F32 => cx.type_f32(), | |
858 | Primitive::F64 => cx.type_f64(), | |
859 | Primitive::Pointer => cx.type_isize(), | |
860 | _ => unreachable!(), | |
861 | } | |
862 | } | |
863 | ||
864 | /// Fix up an input value to work around LLVM bugs. | |
a2a8927a XL |
865 | fn llvm_fixup_input<'ll, 'tcx>( |
866 | bx: &mut Builder<'_, 'll, 'tcx>, | |
f9f354fc XL |
867 | mut value: &'ll Value, |
868 | reg: InlineAsmRegClass, | |
869 | layout: &TyAndLayout<'tcx>, | |
870 | ) -> &'ll Value { | |
c295e0f8 | 871 | match (reg, layout.abi) { |
f9f354fc | 872 | (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => { |
04454e1e | 873 | if let Primitive::Int(Integer::I8, _) = s.primitive() { |
f9f354fc XL |
874 | let vec_ty = bx.cx.type_vector(bx.cx.type_i8(), 8); |
875 | bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0)) | |
876 | } else { | |
877 | value | |
878 | } | |
879 | } | |
880 | (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) => { | |
881 | let elem_ty = llvm_asm_scalar_type(bx.cx, s); | |
882 | let count = 16 / layout.size.bytes(); | |
883 | let vec_ty = bx.cx.type_vector(elem_ty, count); | |
04454e1e | 884 | if let Primitive::Pointer = s.primitive() { |
f9f354fc XL |
885 | value = bx.ptrtoint(value, bx.cx.type_isize()); |
886 | } | |
887 | bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0)) | |
888 | } | |
889 | ( | |
890 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), | |
891 | Abi::Vector { element, count }, | |
892 | ) if layout.size.bytes() == 8 => { | |
893 | let elem_ty = llvm_asm_scalar_type(bx.cx, element); | |
c295e0f8 | 894 | let vec_ty = bx.cx.type_vector(elem_ty, count); |
f9f354fc XL |
895 | let indices: Vec<_> = (0..count * 2).map(|x| bx.const_i32(x as i32)).collect(); |
896 | bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices)) | |
897 | } | |
898 | (InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd), Abi::Scalar(s)) | |
04454e1e | 899 | if s.primitive() == Primitive::F64 => |
f9f354fc XL |
900 | { |
901 | bx.bitcast(value, bx.cx.type_i64()) | |
902 | } | |
903 | ( | |
904 | InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg), | |
905 | Abi::Vector { .. }, | |
906 | ) if layout.size.bytes() == 64 => bx.bitcast(value, bx.cx.type_vector(bx.cx.type_f64(), 8)), | |
3dfed10e XL |
907 | ( |
908 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16), | |
909 | Abi::Scalar(s), | |
910 | ) => { | |
04454e1e | 911 | if let Primitive::Int(Integer::I32, _) = s.primitive() { |
3dfed10e XL |
912 | bx.bitcast(value, bx.cx.type_f32()) |
913 | } else { | |
914 | value | |
915 | } | |
916 | } | |
f9f354fc XL |
917 | ( |
918 | InlineAsmRegClass::Arm( | |
3dfed10e | 919 | ArmInlineAsmRegClass::dreg |
f9f354fc | 920 | | ArmInlineAsmRegClass::dreg_low8 |
3dfed10e | 921 | | ArmInlineAsmRegClass::dreg_low16, |
f9f354fc XL |
922 | ), |
923 | Abi::Scalar(s), | |
924 | ) => { | |
04454e1e | 925 | if let Primitive::Int(Integer::I64, _) = s.primitive() { |
3dfed10e | 926 | bx.bitcast(value, bx.cx.type_f64()) |
f9f354fc XL |
927 | } else { |
928 | value | |
929 | } | |
930 | } | |
04454e1e FG |
931 | (InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg), Abi::Scalar(s)) => { |
932 | match s.primitive() { | |
933 | // MIPS only supports register-length arithmetics. | |
934 | Primitive::Int(Integer::I8 | Integer::I16, _) => bx.zext(value, bx.cx.type_i32()), | |
935 | Primitive::F32 => bx.bitcast(value, bx.cx.type_i32()), | |
936 | Primitive::F64 => bx.bitcast(value, bx.cx.type_i64()), | |
937 | _ => value, | |
938 | } | |
939 | } | |
f9f354fc XL |
940 | _ => value, |
941 | } | |
942 | } | |
943 | ||
944 | /// Fix up an output value to work around LLVM bugs. | |
a2a8927a XL |
945 | fn llvm_fixup_output<'ll, 'tcx>( |
946 | bx: &mut Builder<'_, 'll, 'tcx>, | |
f9f354fc XL |
947 | mut value: &'ll Value, |
948 | reg: InlineAsmRegClass, | |
949 | layout: &TyAndLayout<'tcx>, | |
950 | ) -> &'ll Value { | |
c295e0f8 | 951 | match (reg, layout.abi) { |
f9f354fc | 952 | (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => { |
04454e1e | 953 | if let Primitive::Int(Integer::I8, _) = s.primitive() { |
f9f354fc XL |
954 | bx.extract_element(value, bx.const_i32(0)) |
955 | } else { | |
956 | value | |
957 | } | |
958 | } | |
959 | (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) => { | |
960 | value = bx.extract_element(value, bx.const_i32(0)); | |
04454e1e | 961 | if let Primitive::Pointer = s.primitive() { |
f9f354fc XL |
962 | value = bx.inttoptr(value, layout.llvm_type(bx.cx)); |
963 | } | |
964 | value | |
965 | } | |
966 | ( | |
967 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), | |
968 | Abi::Vector { element, count }, | |
969 | ) if layout.size.bytes() == 8 => { | |
970 | let elem_ty = llvm_asm_scalar_type(bx.cx, element); | |
c295e0f8 XL |
971 | let vec_ty = bx.cx.type_vector(elem_ty, count * 2); |
972 | let indices: Vec<_> = (0..count).map(|x| bx.const_i32(x as i32)).collect(); | |
f9f354fc XL |
973 | bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices)) |
974 | } | |
975 | (InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd), Abi::Scalar(s)) | |
04454e1e | 976 | if s.primitive() == Primitive::F64 => |
f9f354fc XL |
977 | { |
978 | bx.bitcast(value, bx.cx.type_f64()) | |
979 | } | |
980 | ( | |
981 | InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg), | |
982 | Abi::Vector { .. }, | |
983 | ) if layout.size.bytes() == 64 => bx.bitcast(value, layout.llvm_type(bx.cx)), | |
3dfed10e XL |
984 | ( |
985 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16), | |
986 | Abi::Scalar(s), | |
987 | ) => { | |
04454e1e | 988 | if let Primitive::Int(Integer::I32, _) = s.primitive() { |
3dfed10e XL |
989 | bx.bitcast(value, bx.cx.type_i32()) |
990 | } else { | |
991 | value | |
992 | } | |
993 | } | |
f9f354fc XL |
994 | ( |
995 | InlineAsmRegClass::Arm( | |
3dfed10e | 996 | ArmInlineAsmRegClass::dreg |
f9f354fc | 997 | | ArmInlineAsmRegClass::dreg_low8 |
3dfed10e | 998 | | ArmInlineAsmRegClass::dreg_low16, |
f9f354fc XL |
999 | ), |
1000 | Abi::Scalar(s), | |
1001 | ) => { | |
04454e1e | 1002 | if let Primitive::Int(Integer::I64, _) = s.primitive() { |
3dfed10e | 1003 | bx.bitcast(value, bx.cx.type_i64()) |
f9f354fc XL |
1004 | } else { |
1005 | value | |
1006 | } | |
1007 | } | |
04454e1e FG |
1008 | (InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg), Abi::Scalar(s)) => { |
1009 | match s.primitive() { | |
1010 | // MIPS only supports register-length arithmetics. | |
1011 | Primitive::Int(Integer::I8, _) => bx.trunc(value, bx.cx.type_i8()), | |
1012 | Primitive::Int(Integer::I16, _) => bx.trunc(value, bx.cx.type_i16()), | |
1013 | Primitive::F32 => bx.bitcast(value, bx.cx.type_f32()), | |
1014 | Primitive::F64 => bx.bitcast(value, bx.cx.type_f64()), | |
1015 | _ => value, | |
1016 | } | |
1017 | } | |
f9f354fc XL |
1018 | _ => value, |
1019 | } | |
1020 | } | |
1021 | ||
1022 | /// Output type to use for llvm_fixup_output. | |
a2a8927a | 1023 | fn llvm_fixup_output_type<'ll, 'tcx>( |
f9f354fc XL |
1024 | cx: &CodegenCx<'ll, 'tcx>, |
1025 | reg: InlineAsmRegClass, | |
1026 | layout: &TyAndLayout<'tcx>, | |
1027 | ) -> &'ll Type { | |
c295e0f8 | 1028 | match (reg, layout.abi) { |
f9f354fc | 1029 | (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => { |
04454e1e | 1030 | if let Primitive::Int(Integer::I8, _) = s.primitive() { |
f9f354fc XL |
1031 | cx.type_vector(cx.type_i8(), 8) |
1032 | } else { | |
1033 | layout.llvm_type(cx) | |
1034 | } | |
1035 | } | |
1036 | (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) => { | |
1037 | let elem_ty = llvm_asm_scalar_type(cx, s); | |
1038 | let count = 16 / layout.size.bytes(); | |
1039 | cx.type_vector(elem_ty, count) | |
1040 | } | |
1041 | ( | |
1042 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), | |
1043 | Abi::Vector { element, count }, | |
1044 | ) if layout.size.bytes() == 8 => { | |
1045 | let elem_ty = llvm_asm_scalar_type(cx, element); | |
1046 | cx.type_vector(elem_ty, count * 2) | |
1047 | } | |
1048 | (InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd), Abi::Scalar(s)) | |
04454e1e | 1049 | if s.primitive() == Primitive::F64 => |
f9f354fc XL |
1050 | { |
1051 | cx.type_i64() | |
1052 | } | |
1053 | ( | |
1054 | InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg), | |
1055 | Abi::Vector { .. }, | |
1056 | ) if layout.size.bytes() == 64 => cx.type_vector(cx.type_f64(), 8), | |
3dfed10e XL |
1057 | ( |
1058 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16), | |
1059 | Abi::Scalar(s), | |
1060 | ) => { | |
04454e1e | 1061 | if let Primitive::Int(Integer::I32, _) = s.primitive() { |
3dfed10e XL |
1062 | cx.type_f32() |
1063 | } else { | |
1064 | layout.llvm_type(cx) | |
1065 | } | |
1066 | } | |
f9f354fc XL |
1067 | ( |
1068 | InlineAsmRegClass::Arm( | |
3dfed10e | 1069 | ArmInlineAsmRegClass::dreg |
f9f354fc | 1070 | | ArmInlineAsmRegClass::dreg_low8 |
3dfed10e | 1071 | | ArmInlineAsmRegClass::dreg_low16, |
f9f354fc XL |
1072 | ), |
1073 | Abi::Scalar(s), | |
1074 | ) => { | |
04454e1e | 1075 | if let Primitive::Int(Integer::I64, _) = s.primitive() { |
3dfed10e | 1076 | cx.type_f64() |
f9f354fc XL |
1077 | } else { |
1078 | layout.llvm_type(cx) | |
1079 | } | |
1080 | } | |
04454e1e FG |
1081 | (InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg), Abi::Scalar(s)) => { |
1082 | match s.primitive() { | |
1083 | // MIPS only supports register-length arithmetics. | |
1084 | Primitive::Int(Integer::I8 | Integer::I16, _) => cx.type_i32(), | |
1085 | Primitive::F32 => cx.type_i32(), | |
1086 | Primitive::F64 => cx.type_i64(), | |
1087 | _ => layout.llvm_type(cx), | |
1088 | } | |
1089 | } | |
f9f354fc XL |
1090 | _ => layout.llvm_type(cx), |
1091 | } | |
1092 | } |