6 clippy
::match_same_arms
,
7 clippy
::nonminimal_bool
,
11 clippy
::useless_format
14 use std
::collections
::{BTreeMap, HashMap}
;
16 use serde
::Deserialize
;
18 const PRINT_INSTRUCTION_VIOLATIONS
: bool
= false;
19 const PRINT_MISSING_LISTS
: bool
= false;
20 const PRINT_MISSING_LISTS_MARKDOWN
: bool
= false;
24 arguments
: &'
static [&'
static Type
],
25 ret
: Option
<&'
static Type
>,
26 target_feature
: Option
<&'
static str>,
27 instrs
: &'
static [&'
static str],
29 required_const
: &'
static [usize],
33 static F32
: Type
= Type
::PrimFloat(32);
34 static F64
: Type
= Type
::PrimFloat(64);
35 static I16
: Type
= Type
::PrimSigned(16);
36 static I32
: Type
= Type
::PrimSigned(32);
37 static I64
: Type
= Type
::PrimSigned(64);
38 static I8
: Type
= Type
::PrimSigned(8);
39 static U16
: Type
= Type
::PrimUnsigned(16);
40 static U32
: Type
= Type
::PrimUnsigned(32);
41 static U64
: Type
= Type
::PrimUnsigned(64);
42 static U128
: Type
= Type
::PrimUnsigned(128);
43 static U8
: Type
= Type
::PrimUnsigned(8);
44 static ORDERING
: Type
= Type
::Ordering
;
46 static M64
: Type
= Type
::M64
;
47 static M128
: Type
= Type
::M128
;
48 static M128I
: Type
= Type
::M128I
;
49 static M128D
: Type
= Type
::M128D
;
50 static M256
: Type
= Type
::M256
;
51 static M256I
: Type
= Type
::M256I
;
52 static M256D
: Type
= Type
::M256D
;
53 static M512
: Type
= Type
::M512
;
54 static M512I
: Type
= Type
::M512I
;
55 static M512D
: Type
= Type
::M512D
;
56 static MMASK8
: Type
= Type
::MMASK8
;
57 static MMASK16
: Type
= Type
::MMASK16
;
59 static TUPLE
: Type
= Type
::Tuple
;
60 static CPUID
: Type
= Type
::CpuidResult
;
61 static NEVER
: Type
= Type
::Never
;
68 MutPtr(&'
static Type
),
69 ConstPtr(&'
static Type
),
88 stdarch_verify
::x86_functions
!(static FUNCTIONS
);
90 #[derive(Deserialize)]
92 #[serde(rename = "intrinsic", default)]
93 intrinsics
: Vec
<Intrinsic
>,
96 #[derive(Deserialize)]
100 #[serde(rename = "CPUID", default)]
102 #[serde(rename = "parameter", default)]
103 parameters
: Vec
<Parameter
>,
105 instruction
: Vec
<Instruction
>,
108 #[derive(Deserialize)]
110 #[serde(rename = "type")]
114 #[derive(Deserialize, Debug)]
120 ($
($t
:tt
)*) => (return Err(format
!($
($t
)*)))
124 fn verify_all_signatures() {
125 // This XML document was downloaded from Intel's site. To update this you
126 // can visit intel's intrinsics guide online documentation:
128 // https://software.intel.com/sites/landingpage/IntrinsicsGuide/#
130 // Open up the network console and you'll see an xml file was downloaded
131 // (currently called data-3.4.xml). That's the file we downloaded
133 let xml
= include_bytes
!("../x86-intel.xml");
136 let data
: Data
= serde_xml_rs
::from_reader(xml
).expect("failed to deserialize xml");
137 let mut map
= HashMap
::new();
138 for intrinsic
in &data
.intrinsics
{
139 map
.entry(&intrinsic
.name
[..])
140 .or_insert_with(Vec
::new
)
144 let mut all_valid
= true;
145 'outer
: for rust
in FUNCTIONS
{
147 // FIXME: this list should be almost empty
162 "_MM_GET_EXCEPTION_MASK",
163 "_MM_GET_EXCEPTION_STATE",
164 "_MM_GET_FLUSH_ZERO_MODE",
165 "_MM_GET_ROUNDING_MODE",
166 "_MM_SET_EXCEPTION_MASK",
167 "_MM_SET_EXCEPTION_STATE",
168 "_MM_SET_FLUSH_ZERO_MODE",
169 "_MM_SET_ROUNDING_MODE",
201 "_mm_undefined_si128",
206 "_mm256_castps128_ps256",
207 "_mm256_castpd128_pd256",
208 "_mm256_castsi128_si256",
209 "_mm256_undefined_ps",
210 "_mm256_undefined_pd",
211 "_mm256_undefined_si256",
214 "_mm512_setzero_si512",
252 "_mm256_shuffle_epi32",
253 "_mm256_bslli_epi128",
254 "_mm256_bsrli_epi128",
255 "_mm256_unpackhi_epi8",
256 "_mm256_unpacklo_epi8",
257 "_mm256_unpackhi_epi16",
258 "_mm256_unpacklo_epi16",
259 "_mm256_unpackhi_epi32",
260 "_mm256_unpacklo_epi32",
261 "_mm256_unpackhi_epi64",
262 "_mm256_unpacklo_epi64",
269 "_mm_cvtsi64x_si128",
270 "_mm_cvtsi128_si64x",
280 if !skip
.contains(&rust
.name
) {
282 "missing run-time test named `test_{}` for `{}`",
284 let mut id
= rust
.name
;
285 while id
.starts_with('_'
) {
297 // These aren't defined by Intel but they're defined by what appears
298 // to be all other compilers. For more information see
299 // rust-lang/stdarch#307, and otherwise these signatures
300 // have all been manually verified.
306 // Not listed with intel, but manually verified
308 // The UD2 intrinsic is not defined by Intel, but it was agreed on
309 // in the RFC Issue 2512:
310 // https://github.com/rust-lang/rfcs/issues/2512
313 // Intel requires the mask argument for _mm_shuffle_ps to be an
314 // unsigned integer, but all other _mm_shuffle_.. intrinsics
315 // take a signed-integer. This breaks `_MM_SHUFFLE` for
317 "_mm_shuffle_ps" => continue,
321 // these are all AMD-specific intrinsics
322 if let Some(feature
) = rust
.target_feature
{
323 if feature
.contains("sse4a") || feature
.contains("tbm") {
328 let intel
= match map
.remove(rust
.name
) {
330 None
=> panic
!("missing intel definition for {}", rust
.name
),
333 let mut errors
= Vec
::new();
335 match matches(rust
, intel
) {
336 Ok(()) => continue 'outer
,
337 Err(e
) => errors
.push(e
),
340 println
!("failed to verify `{}`", rust
.name
);
341 for error
in errors
{
342 println
!(" * {}", error
);
348 let mut missing
= BTreeMap
::new();
349 for (name
, intel
) in &map
{
350 // currently focused mainly on missing SIMD intrinsics, but there's
351 // definitely some other assorted ones that we're missing.
352 if !name
.starts_with("_mm") {
356 // we'll get to avx-512 later
357 // let avx512 = intel.iter().any(|i| {
358 // i.name.starts_with("_mm512") || i.cpuid.iter().any(|c| {
369 .or_insert_with(Vec
::new
)
374 // generate a bulleted list of missing intrinsics
375 if PRINT_MISSING_LISTS
|| PRINT_MISSING_LISTS_MARKDOWN
{
376 for (k
, v
) in missing
{
377 if PRINT_MISSING_LISTS_MARKDOWN
{
378 println
!("\n<details><summary>{:?}</summary><p>\n", k
);
381 "https://software.intel.com/sites/landingpage\
382 /IntrinsicsGuide/#text={}&expand=5236",
385 println
!(" * [ ] [`{}`]({})", intel
.name
, url
);
387 println
!("</p></details>\n");
389 println
!("\n{:?}\n", k
);
391 println
!("\t{}", intel
.name
);
398 fn matches(rust
: &Function
, intel
: &Intrinsic
) -> Result
<(), String
> {
399 // Verify that all `#[target_feature]` annotations are correct,
400 // ensuring that we've actually enabled the right instruction
401 // set for this intrinsic.
403 "_bswap" | "_bswap64" => {}
405 // These don't actually have a target feature unlike their brethren with
406 // the `x` inside the name which requires adx
407 "_addcarry_u32" | "_addcarry_u64" | "_subborrow_u32" | "_subborrow_u64" => {}
412 | "_bittestandcomplement"
415 | "_bittestandreset64"
416 | "_bittestandcomplement64" => {}
419 if intel
.cpuid
.is_empty() {
420 bail
!("missing cpuid for {}", rust
.name
);
425 for cpuid
in &intel
.cpuid
{
426 // The pause intrinsic is in the SSE2 module, but it is backwards
427 // compatible with CPUs without SSE2, and it therefore does not need the
428 // target-feature attribute.
429 if rust
.name
== "_mm_pause" {
432 // this is needed by _xsave and probably some related intrinsics,
433 // but let's just skip it for now.
438 // these flags on the rdtsc/rtdscp intrinsics we don't test for right
439 // now, but we may wish to add these one day!
441 // For more info see #308
442 if *cpuid
== "TSC" || *cpuid
== "RDTSCP" {
448 .flat_map(|c
| c
.to_lowercase())
449 .collect
::<String
>();
451 // Fix mismatching feature names:
452 let fixup_cpuid
= |cpuid
: String
| match cpuid
.as_ref() {
453 // The XML file names IFMA as "avx512ifma52", while Rust calls
455 "avx512ifma52" => String
::from("avx512ifma"),
456 // See: https://github.com/rust-lang/stdarch/issues/738
457 // The intrinsics guide calls `f16c` `fp16c` in disagreement with
458 // Intel's architecture manuals.
459 "fp16c" => String
::from("f16c"),
462 let fixed_cpuid
= fixup_cpuid(cpuid
);
464 let rust_feature
= rust
466 .expect(&format
!("no target feature listed for {}", rust
.name
));
468 if rust_feature
.contains(&fixed_cpuid
) {
472 "intel cpuid `{}` not in `{}` for {}",
479 if PRINT_INSTRUCTION_VIOLATIONS
{
480 if rust
.instrs
.is_empty() {
481 if !intel
.instruction
.is_empty() {
483 "instruction not listed for `{}`, but intel lists {:?}",
484 rust
.name
, intel
.instruction
488 // If intel doesn't list any instructions and we do then don't
489 // bother trying to look for instructions in intel, we've just got
490 // some extra assertions on our end.
491 } else if !intel
.instruction
.is_empty() {
492 for instr
in rust
.instrs
{
493 let asserting
= intel
.instruction
.iter().any(|a
| a
.name
.starts_with(instr
));
496 "intel failed to list `{}` as an instruction for `{}`",
504 // Make sure we've got the right return type.
505 if let Some(t
) = rust
.ret
{
506 equate(t
, &intel
.rettype
, rust
.name
, false)?
;
507 } else if intel
.rettype
!= "" && intel
.rettype
!= "void" {
509 "{} returns `{}` with intel, void in rust",
515 // If there's no arguments on Rust's side intel may list one "void"
516 // argument, so handle that here.
517 if rust
.arguments
.is_empty() && intel
.parameters
.len() == 1 {
518 if intel
.parameters
[0].type_
!= "void" {
519 bail
!("rust has 0 arguments, intel has one for")
522 // Otherwise we want all parameters to be exactly the same
523 if rust
.arguments
.len() != intel
.parameters
.len() {
524 bail
!("wrong number of arguments on {}", rust
.name
)
526 for (i
, (a
, b
)) in intel
.parameters
.iter().zip(rust
.arguments
).enumerate() {
527 let is_const
= rust
.required_const
.contains(&i
);
528 equate(b
, &a
.type_
, &intel
.name
, is_const
)?
;
537 .any(|arg
| match *arg
{
538 Type
::PrimSigned(64) | Type
::PrimUnsigned(64) => true,
541 let any_i64_exempt
= match rust
.name
{
542 // These intrinsics have all been manually verified against Clang's
543 // headers to be available on x86, and the u64 arguments seem
545 "_xsave" | "_xrstor" | "_xsetbv" | "_xgetbv" | "_xsaveopt" | "_xsavec" | "_xsaves"
546 | "_xrstors" => true,
548 // Apparently all of clang/msvc/gcc accept these intrinsics on
549 // 32-bit, so let's do the same
550 "_mm_set_epi64x" | "_mm_set1_epi64x" | "_mm256_set_epi64x" | "_mm256_setr_epi64x"
551 | "_mm256_set1_epi64x" | "_mm512_set1_epi64" => true,
553 // These return a 64-bit argument but they're assembled from other
554 // 32-bit registers, so these work on 32-bit just fine. See #308 for
556 "_rdtsc" | "__rdtscp" => true,
560 if any_i64
&& !any_i64_exempt
&& !rust
.file
.contains("x86_64") {
562 "intrinsic `{}` uses a 64-bit bare type but may be \
563 available on 32-bit platforms",
570 fn equate(t
: &Type
, intel
: &str, intrinsic
: &str, is_const
: bool
) -> Result
<(), String
> {
571 // Make pointer adjacent to the type: float * foo => float* foo
572 let mut intel
= intel
.replace(" *", "*");
573 // Make mutability modifier adjacent to the pointer:
574 // float const * foo => float const* foo
575 intel
= intel
.replace("const *", "const*");
576 // Normalize mutability modifier to after the type:
577 // const float* foo => float const*
578 if intel
.starts_with("const") && intel
.ends_with("*") {
579 intel
= intel
.replace("const ", "");
580 intel
= intel
.replace("*", " const*");
582 let require_const
= || {
586 Err(format
!("argument required to be const but isn't"))
588 match (t
, &intel
[..]) {
589 (&Type
::PrimFloat(32), "float") => {}
590 (&Type
::PrimFloat(64), "double") => {}
591 (&Type
::PrimSigned(16), "__int16") => {}
592 (&Type
::PrimSigned(16), "short") => {}
593 (&Type
::PrimSigned(32), "__int32") => {}
594 (&Type
::PrimSigned(32), "const int") => require_const()?
,
595 (&Type
::PrimSigned(32), "int") => {}
596 (&Type
::PrimSigned(64), "__int64") => {}
597 (&Type
::PrimSigned(64), "long long") => {}
598 (&Type
::PrimSigned(8), "__int8") => {}
599 (&Type
::PrimSigned(8), "char") => {}
600 (&Type
::PrimUnsigned(16), "unsigned short") => {}
601 (&Type
::PrimUnsigned(32), "unsigned int") => {}
602 (&Type
::PrimUnsigned(32), "const unsigned int") => {}
603 (&Type
::PrimUnsigned(64), "unsigned __int64") => {}
604 (&Type
::PrimUnsigned(8), "unsigned char") => {}
605 (&Type
::M64
, "__m64") => {}
606 (&Type
::M128
, "__m128") => {}
607 (&Type
::M128I
, "__m128i") => {}
608 (&Type
::M128D
, "__m128d") => {}
609 (&Type
::M256
, "__m256") => {}
610 (&Type
::M256I
, "__m256i") => {}
611 (&Type
::M256D
, "__m256d") => {}
612 (&Type
::M512
, "__m512") => {}
613 (&Type
::M512I
, "__m512i") => {}
614 (&Type
::M512D
, "__m512d") => {}
616 (&Type
::MutPtr(&Type
::PrimFloat(32)), "float*") => {}
617 (&Type
::MutPtr(&Type
::PrimFloat(64)), "double*") => {}
618 (&Type
::MutPtr(&Type
::PrimSigned(32)), "int*") => {}
619 (&Type
::MutPtr(&Type
::PrimSigned(32)), "__int32*") => {}
620 (&Type
::MutPtr(&Type
::PrimSigned(64)), "__int64*") => {}
621 (&Type
::MutPtr(&Type
::PrimSigned(8)), "char*") => {}
622 (&Type
::MutPtr(&Type
::PrimUnsigned(16)), "unsigned short*") => {}
623 (&Type
::MutPtr(&Type
::PrimUnsigned(32)), "unsigned int*") => {}
624 (&Type
::MutPtr(&Type
::PrimUnsigned(64)), "unsigned __int64*") => {}
625 (&Type
::MutPtr(&Type
::PrimUnsigned(8)), "void*") => {}
626 (&Type
::MutPtr(&Type
::M64
), "__m64*") => {}
627 (&Type
::MutPtr(&Type
::M128
), "__m128*") => {}
628 (&Type
::MutPtr(&Type
::M128I
), "__m128i*") => {}
629 (&Type
::MutPtr(&Type
::M128D
), "__m128d*") => {}
630 (&Type
::MutPtr(&Type
::M256
), "__m256*") => {}
631 (&Type
::MutPtr(&Type
::M256I
), "__m256i*") => {}
632 (&Type
::MutPtr(&Type
::M256D
), "__m256d*") => {}
633 (&Type
::MutPtr(&Type
::M512
), "__m512*") => {}
634 (&Type
::MutPtr(&Type
::M512I
), "__m512i*") => {}
635 (&Type
::MutPtr(&Type
::M512D
), "__m512d*") => {}
637 (&Type
::ConstPtr(&Type
::PrimFloat(32)), "float const*") => {}
638 (&Type
::ConstPtr(&Type
::PrimFloat(64)), "double const*") => {}
639 (&Type
::ConstPtr(&Type
::PrimSigned(32)), "int const*") => {}
640 (&Type
::ConstPtr(&Type
::PrimSigned(32)), "__int32 const*") => {}
641 (&Type
::ConstPtr(&Type
::PrimSigned(64)), "__int64 const*") => {}
642 (&Type
::ConstPtr(&Type
::PrimSigned(8)), "char const*") => {}
643 (&Type
::ConstPtr(&Type
::PrimUnsigned(16)), "unsigned short const*") => {}
644 (&Type
::ConstPtr(&Type
::PrimUnsigned(32)), "unsigned int const*") => {}
645 (&Type
::ConstPtr(&Type
::PrimUnsigned(64)), "unsigned __int64 const*") => {}
646 (&Type
::ConstPtr(&Type
::PrimUnsigned(8)), "void const*") => {}
647 (&Type
::ConstPtr(&Type
::M64
), "__m64 const*") => {}
648 (&Type
::ConstPtr(&Type
::M128
), "__m128 const*") => {}
649 (&Type
::ConstPtr(&Type
::M128I
), "__m128i const*") => {}
650 (&Type
::ConstPtr(&Type
::M128D
), "__m128d const*") => {}
651 (&Type
::ConstPtr(&Type
::M256
), "__m256 const*") => {}
652 (&Type
::ConstPtr(&Type
::M256I
), "__m256i const*") => {}
653 (&Type
::ConstPtr(&Type
::M256D
), "__m256d const*") => {}
654 (&Type
::ConstPtr(&Type
::M512
), "__m512 const*") => {}
655 (&Type
::ConstPtr(&Type
::M512I
), "__m512i const*") => {}
656 (&Type
::ConstPtr(&Type
::M512D
), "__m512d const*") => {}
658 (&Type
::MMASK8
, "__mmask8") => {}
659 (&Type
::MMASK16
, "__mmask16") => {}
661 // This is a macro (?) in C which seems to mutate its arguments, but
662 // that means that we're taking pointers to arguments in rust
663 // as we're not exposing it as a macro.
664 (&Type
::MutPtr(&Type
::M128
), "__m128") if intrinsic
== "_MM_TRANSPOSE4_PS" => {}
666 // The _rdtsc intrinsic uses a __int64 return type, but this is a bug in
667 // the intrinsics guide: https://github.com/rust-lang/stdarch/issues/559
668 // We have manually fixed the bug by changing the return type to `u64`.
669 (&Type
::PrimUnsigned(64), "__int64") if intrinsic
== "_rdtsc" => {}
671 // The _bittest and _bittest64 intrinsics takes a mutable pointer in the
672 // intrinsics guide even though it never writes through the pointer:
673 (&Type
::ConstPtr(&Type
::PrimSigned(32)), "__int32*") if intrinsic
== "_bittest" => {}
674 (&Type
::ConstPtr(&Type
::PrimSigned(64)), "__int64*") if intrinsic
== "_bittest64" => {}
675 // The _xrstor, _fxrstor, _xrstor64, _fxrstor64 intrinsics take a
676 // mutable pointer in the intrinsics guide even though they never write
677 // through the pointer:
678 (&Type
::ConstPtr(&Type
::PrimUnsigned(8)), "void*")
679 if intrinsic
== "_xrstor"
680 || intrinsic
== "_xrstor64"
681 || intrinsic
== "_fxrstor"
682 || intrinsic
== "_fxrstor64" => {}
685 "failed to equate: `{}` and {:?} for {}",